| ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py |
| ; RUN: opt -passes=lower-matrix-intrinsics -fuse-matrix-use-loops -fuse-matrix-tile-size=2 -matrix-allow-contract -force-fuse-matrix -verify-dom-info %s -S | FileCheck %s |
| |
| ; REQUIRES: aarch64-registered-target |
| |
| target datalayout = "e-m:o-i64:64-f80:128-n8:16:32:64-S128" |
| target triple = "aarch64-apple-ios" |
| |
| define void @multiply_all_volatile(ptr noalias %A, ptr noalias %B, ptr noalias %C) { |
| ; CHECK-LABEL: @multiply_all_volatile( |
| ; CHECK-NEXT: entry: |
| ; CHECK-NEXT: br label [[COLS_HEADER:%.*]] |
| ; CHECK: cols.header: |
| ; CHECK-NEXT: [[COLS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[COLS_STEP:%.*]], [[COLS_LATCH:%.*]] ] |
| ; CHECK-NEXT: br label [[COLS_BODY:%.*]] |
| ; CHECK: cols.body: |
| ; CHECK-NEXT: br label [[ROWS_HEADER:%.*]] |
| ; CHECK: rows.header: |
| ; CHECK-NEXT: [[ROWS_IV:%.*]] = phi i64 [ 0, [[COLS_BODY]] ], [ [[ROWS_STEP:%.*]], [[ROWS_LATCH:%.*]] ] |
| ; CHECK-NEXT: br label [[ROWS_BODY:%.*]] |
| ; CHECK: rows.body: |
| ; CHECK-NEXT: br label [[INNER_HEADER:%.*]] |
| ; CHECK: inner.header: |
| ; CHECK-NEXT: [[INNER_IV:%.*]] = phi i64 [ 0, [[ROWS_BODY]] ], [ [[INNER_STEP:%.*]], [[INNER_LATCH:%.*]] ] |
| ; CHECK-NEXT: [[RESULT_VEC_0:%.*]] = phi <2 x double> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP11:%.*]], [[INNER_LATCH]] ] |
| ; CHECK-NEXT: [[RESULT_VEC_1:%.*]] = phi <2 x double> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP17:%.*]], [[INNER_LATCH]] ] |
| ; CHECK-NEXT: br label [[INNER_BODY:%.*]] |
| ; CHECK: inner.body: |
| ; CHECK-NEXT: [[TMP0:%.*]] = mul i64 [[INNER_IV]], 2 |
| ; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[TMP0]], [[ROWS_IV]] |
| ; CHECK-NEXT: [[TMP2:%.*]] = getelementptr double, ptr [[A:%.*]], i64 [[TMP1]] |
| ; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, ptr [[TMP2]], align 8 |
| ; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, ptr [[TMP2]], i64 2 |
| ; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x double>, ptr [[VEC_GEP]], align 8 |
| ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[COLS_IV]], 2 |
| ; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[TMP3]], [[INNER_IV]] |
| ; CHECK-NEXT: [[TMP5:%.*]] = getelementptr double, ptr [[B:%.*]], i64 [[TMP4]] |
| ; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x double>, ptr [[TMP5]], align 8 |
| ; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr double, ptr [[TMP5]], i64 2 |
| ; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x double>, ptr [[VEC_GEP3]], align 8 |
| ; CHECK-NEXT: [[BLOCK:%.*]] = shufflevector <2 x double> [[RESULT_VEC_0]], <2 x double> poison, <2 x i32> <i32 0, i32 1> |
| ; CHECK-NEXT: [[BLOCK5:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> poison, <2 x i32> <i32 0, i32 1> |
| ; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x double> [[COL_LOAD2]], i64 0 |
| ; CHECK-NEXT: [[SPLAT_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[TMP6]], i64 0 |
| ; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT]], <2 x double> poison, <2 x i32> zeroinitializer |
| ; CHECK-NEXT: [[TMP7:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK5]], <2 x double> [[SPLAT_SPLAT]], <2 x double> [[BLOCK]]) |
| ; CHECK-NEXT: [[BLOCK6:%.*]] = shufflevector <2 x double> [[COL_LOAD1]], <2 x double> poison, <2 x i32> <i32 0, i32 1> |
| ; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x double> [[COL_LOAD2]], i64 1 |
| ; CHECK-NEXT: [[SPLAT_SPLATINSERT7:%.*]] = insertelement <2 x double> poison, double [[TMP8]], i64 0 |
| ; CHECK-NEXT: [[SPLAT_SPLAT8:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT7]], <2 x double> poison, <2 x i32> zeroinitializer |
| ; CHECK-NEXT: [[TMP9:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK6]], <2 x double> [[SPLAT_SPLAT8]], <2 x double> [[TMP7]]) |
| ; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <2 x double> [[TMP9]], <2 x double> poison, <2 x i32> <i32 0, i32 1> |
| ; CHECK-NEXT: [[TMP11]] = shufflevector <2 x double> [[RESULT_VEC_0]], <2 x double> [[TMP10]], <2 x i32> <i32 2, i32 3> |
| ; CHECK-NEXT: [[BLOCK9:%.*]] = shufflevector <2 x double> [[RESULT_VEC_1]], <2 x double> poison, <2 x i32> <i32 0, i32 1> |
| ; CHECK-NEXT: [[BLOCK10:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> poison, <2 x i32> <i32 0, i32 1> |
| ; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x double> [[COL_LOAD4]], i64 0 |
| ; CHECK-NEXT: [[SPLAT_SPLATINSERT11:%.*]] = insertelement <2 x double> poison, double [[TMP12]], i64 0 |
| ; CHECK-NEXT: [[SPLAT_SPLAT12:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT11]], <2 x double> poison, <2 x i32> zeroinitializer |
| ; CHECK-NEXT: [[TMP13:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK10]], <2 x double> [[SPLAT_SPLAT12]], <2 x double> [[BLOCK9]]) |
| ; CHECK-NEXT: [[BLOCK13:%.*]] = shufflevector <2 x double> [[COL_LOAD1]], <2 x double> poison, <2 x i32> <i32 0, i32 1> |
| ; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x double> [[COL_LOAD4]], i64 1 |
| ; CHECK-NEXT: [[SPLAT_SPLATINSERT14:%.*]] = insertelement <2 x double> poison, double [[TMP14]], i64 0 |
| ; CHECK-NEXT: [[SPLAT_SPLAT15:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT14]], <2 x double> poison, <2 x i32> zeroinitializer |
| ; CHECK-NEXT: [[TMP15:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK13]], <2 x double> [[SPLAT_SPLAT15]], <2 x double> [[TMP13]]) |
| ; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <2 x double> [[TMP15]], <2 x double> poison, <2 x i32> <i32 0, i32 1> |
| ; CHECK-NEXT: [[TMP17]] = shufflevector <2 x double> [[RESULT_VEC_1]], <2 x double> [[TMP16]], <2 x i32> <i32 2, i32 3> |
| ; CHECK-NEXT: br label [[INNER_LATCH]] |
| ; CHECK: inner.latch: |
| ; CHECK-NEXT: [[INNER_STEP]] = add i64 [[INNER_IV]], 2 |
| ; CHECK-NEXT: [[INNER_COND:%.*]] = icmp ne i64 [[INNER_STEP]], 2 |
| ; CHECK-NEXT: br i1 [[INNER_COND]], label [[INNER_HEADER]], label [[ROWS_LATCH]], !llvm.loop [[LOOP0:![0-9]+]] |
| ; CHECK: rows.latch: |
| ; CHECK-NEXT: [[ROWS_STEP]] = add i64 [[ROWS_IV]], 2 |
| ; CHECK-NEXT: [[ROWS_COND:%.*]] = icmp ne i64 [[ROWS_STEP]], 2 |
| ; CHECK-NEXT: [[TMP18:%.*]] = mul i64 [[COLS_IV]], 2 |
| ; CHECK-NEXT: [[TMP19:%.*]] = add i64 [[TMP18]], [[ROWS_IV]] |
| ; CHECK-NEXT: [[TMP20:%.*]] = getelementptr double, ptr [[C:%.*]], i64 [[TMP19]] |
| ; CHECK-NEXT: store volatile <2 x double> [[TMP11]], ptr [[TMP20]], align 8 |
| ; CHECK-NEXT: [[VEC_GEP16:%.*]] = getelementptr double, ptr [[TMP20]], i64 2 |
| ; CHECK-NEXT: store volatile <2 x double> [[TMP17]], ptr [[VEC_GEP16]], align 8 |
| ; CHECK-NEXT: br i1 [[ROWS_COND]], label [[ROWS_HEADER]], label [[COLS_LATCH]] |
| ; CHECK: cols.latch: |
| ; CHECK-NEXT: [[COLS_STEP]] = add i64 [[COLS_IV]], 2 |
| ; CHECK-NEXT: [[COLS_COND:%.*]] = icmp ne i64 [[COLS_STEP]], 2 |
| ; CHECK-NEXT: br i1 [[COLS_COND]], label [[COLS_HEADER]], label [[CONTINUE:%.*]] |
| ; CHECK: continue: |
| ; CHECK-NEXT: ret void |
| ; |
| |
| |
| entry: |
| %a = load volatile <4 x double>, ptr %A, align 8 |
| %b = load volatile <4 x double>, ptr %B, align 8 |
| |
| %c = call <4 x double> @llvm.matrix.multiply(<4 x double> %a, <4 x double> %b, i32 2, i32 2, i32 2) |
| |
| store volatile <4 x double> %c, ptr %C, align 8 |
| ret void |
| } |
| |
| |
| define void @multiply_load0_volatile(ptr noalias %A, ptr noalias %B, ptr noalias %C) { |
| ; CHECK-LABEL: @multiply_load0_volatile( |
| ; CHECK-NEXT: entry: |
| ; CHECK-NEXT: br label [[COLS_HEADER:%.*]] |
| ; CHECK: cols.header: |
| ; CHECK-NEXT: [[COLS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[COLS_STEP:%.*]], [[COLS_LATCH:%.*]] ] |
| ; CHECK-NEXT: br label [[COLS_BODY:%.*]] |
| ; CHECK: cols.body: |
| ; CHECK-NEXT: br label [[ROWS_HEADER:%.*]] |
| ; CHECK: rows.header: |
| ; CHECK-NEXT: [[ROWS_IV:%.*]] = phi i64 [ 0, [[COLS_BODY]] ], [ [[ROWS_STEP:%.*]], [[ROWS_LATCH:%.*]] ] |
| ; CHECK-NEXT: br label [[ROWS_BODY:%.*]] |
| ; CHECK: rows.body: |
| ; CHECK-NEXT: br label [[INNER_HEADER:%.*]] |
| ; CHECK: inner.header: |
| ; CHECK-NEXT: [[INNER_IV:%.*]] = phi i64 [ 0, [[ROWS_BODY]] ], [ [[INNER_STEP:%.*]], [[INNER_LATCH:%.*]] ] |
| ; CHECK-NEXT: [[RESULT_VEC_0:%.*]] = phi <2 x double> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP11:%.*]], [[INNER_LATCH]] ] |
| ; CHECK-NEXT: [[RESULT_VEC_1:%.*]] = phi <2 x double> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP17:%.*]], [[INNER_LATCH]] ] |
| ; CHECK-NEXT: br label [[INNER_BODY:%.*]] |
| ; CHECK: inner.body: |
| ; CHECK-NEXT: [[TMP0:%.*]] = mul i64 [[INNER_IV]], 2 |
| ; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[TMP0]], [[ROWS_IV]] |
| ; CHECK-NEXT: [[TMP2:%.*]] = getelementptr double, ptr [[A:%.*]], i64 [[TMP1]] |
| ; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, ptr [[TMP2]], align 8 |
| ; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, ptr [[TMP2]], i64 2 |
| ; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x double>, ptr [[VEC_GEP]], align 8 |
| ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[COLS_IV]], 2 |
| ; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[TMP3]], [[INNER_IV]] |
| ; CHECK-NEXT: [[TMP5:%.*]] = getelementptr double, ptr [[B:%.*]], i64 [[TMP4]] |
| ; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x double>, ptr [[TMP5]], align 8 |
| ; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr double, ptr [[TMP5]], i64 2 |
| ; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x double>, ptr [[VEC_GEP3]], align 8 |
| ; CHECK-NEXT: [[BLOCK:%.*]] = shufflevector <2 x double> [[RESULT_VEC_0]], <2 x double> poison, <2 x i32> <i32 0, i32 1> |
| ; CHECK-NEXT: [[BLOCK5:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> poison, <2 x i32> <i32 0, i32 1> |
| ; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x double> [[COL_LOAD2]], i64 0 |
| ; CHECK-NEXT: [[SPLAT_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[TMP6]], i64 0 |
| ; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT]], <2 x double> poison, <2 x i32> zeroinitializer |
| ; CHECK-NEXT: [[TMP7:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK5]], <2 x double> [[SPLAT_SPLAT]], <2 x double> [[BLOCK]]) |
| ; CHECK-NEXT: [[BLOCK6:%.*]] = shufflevector <2 x double> [[COL_LOAD1]], <2 x double> poison, <2 x i32> <i32 0, i32 1> |
| ; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x double> [[COL_LOAD2]], i64 1 |
| ; CHECK-NEXT: [[SPLAT_SPLATINSERT7:%.*]] = insertelement <2 x double> poison, double [[TMP8]], i64 0 |
| ; CHECK-NEXT: [[SPLAT_SPLAT8:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT7]], <2 x double> poison, <2 x i32> zeroinitializer |
| ; CHECK-NEXT: [[TMP9:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK6]], <2 x double> [[SPLAT_SPLAT8]], <2 x double> [[TMP7]]) |
| ; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <2 x double> [[TMP9]], <2 x double> poison, <2 x i32> <i32 0, i32 1> |
| ; CHECK-NEXT: [[TMP11]] = shufflevector <2 x double> [[RESULT_VEC_0]], <2 x double> [[TMP10]], <2 x i32> <i32 2, i32 3> |
| ; CHECK-NEXT: [[BLOCK9:%.*]] = shufflevector <2 x double> [[RESULT_VEC_1]], <2 x double> poison, <2 x i32> <i32 0, i32 1> |
| ; CHECK-NEXT: [[BLOCK10:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> poison, <2 x i32> <i32 0, i32 1> |
| ; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x double> [[COL_LOAD4]], i64 0 |
| ; CHECK-NEXT: [[SPLAT_SPLATINSERT11:%.*]] = insertelement <2 x double> poison, double [[TMP12]], i64 0 |
| ; CHECK-NEXT: [[SPLAT_SPLAT12:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT11]], <2 x double> poison, <2 x i32> zeroinitializer |
| ; CHECK-NEXT: [[TMP13:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK10]], <2 x double> [[SPLAT_SPLAT12]], <2 x double> [[BLOCK9]]) |
| ; CHECK-NEXT: [[BLOCK13:%.*]] = shufflevector <2 x double> [[COL_LOAD1]], <2 x double> poison, <2 x i32> <i32 0, i32 1> |
| ; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x double> [[COL_LOAD4]], i64 1 |
| ; CHECK-NEXT: [[SPLAT_SPLATINSERT14:%.*]] = insertelement <2 x double> poison, double [[TMP14]], i64 0 |
| ; CHECK-NEXT: [[SPLAT_SPLAT15:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT14]], <2 x double> poison, <2 x i32> zeroinitializer |
| ; CHECK-NEXT: [[TMP15:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK13]], <2 x double> [[SPLAT_SPLAT15]], <2 x double> [[TMP13]]) |
| ; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <2 x double> [[TMP15]], <2 x double> poison, <2 x i32> <i32 0, i32 1> |
| ; CHECK-NEXT: [[TMP17]] = shufflevector <2 x double> [[RESULT_VEC_1]], <2 x double> [[TMP16]], <2 x i32> <i32 2, i32 3> |
| ; CHECK-NEXT: br label [[INNER_LATCH]] |
| ; CHECK: inner.latch: |
| ; CHECK-NEXT: [[INNER_STEP]] = add i64 [[INNER_IV]], 2 |
| ; CHECK-NEXT: [[INNER_COND:%.*]] = icmp ne i64 [[INNER_STEP]], 2 |
| ; CHECK-NEXT: br i1 [[INNER_COND]], label [[INNER_HEADER]], label [[ROWS_LATCH]], !llvm.loop [[LOOP2:![0-9]+]] |
| ; CHECK: rows.latch: |
| ; CHECK-NEXT: [[ROWS_STEP]] = add i64 [[ROWS_IV]], 2 |
| ; CHECK-NEXT: [[ROWS_COND:%.*]] = icmp ne i64 [[ROWS_STEP]], 2 |
| ; CHECK-NEXT: [[TMP18:%.*]] = mul i64 [[COLS_IV]], 2 |
| ; CHECK-NEXT: [[TMP19:%.*]] = add i64 [[TMP18]], [[ROWS_IV]] |
| ; CHECK-NEXT: [[TMP20:%.*]] = getelementptr double, ptr [[C:%.*]], i64 [[TMP19]] |
| ; CHECK-NEXT: store <2 x double> [[TMP11]], ptr [[TMP20]], align 8 |
| ; CHECK-NEXT: [[VEC_GEP16:%.*]] = getelementptr double, ptr [[TMP20]], i64 2 |
| ; CHECK-NEXT: store <2 x double> [[TMP17]], ptr [[VEC_GEP16]], align 8 |
| ; CHECK-NEXT: br i1 [[ROWS_COND]], label [[ROWS_HEADER]], label [[COLS_LATCH]] |
| ; CHECK: cols.latch: |
| ; CHECK-NEXT: [[COLS_STEP]] = add i64 [[COLS_IV]], 2 |
| ; CHECK-NEXT: [[COLS_COND:%.*]] = icmp ne i64 [[COLS_STEP]], 2 |
| ; CHECK-NEXT: br i1 [[COLS_COND]], label [[COLS_HEADER]], label [[CONTINUE:%.*]] |
| ; CHECK: continue: |
| ; CHECK-NEXT: ret void |
| ; |
| |
| |
| entry: |
| %a = load volatile <4 x double>, ptr %A, align 8 |
| %b = load <4 x double>, ptr %B, align 8 |
| |
| %c = call <4 x double> @llvm.matrix.multiply(<4 x double> %a, <4 x double> %b, i32 2, i32 2, i32 2) |
| |
| store <4 x double> %c, ptr %C, align 8 |
| ret void |
| } |
| |
| define void @multiply_load1_volatile(ptr noalias %A, ptr noalias %B, ptr noalias %C) { |
| ; CHECK-LABEL: @multiply_load1_volatile( |
| ; CHECK-NEXT: entry: |
| ; CHECK-NEXT: br label [[COLS_HEADER:%.*]] |
| ; CHECK: cols.header: |
| ; CHECK-NEXT: [[COLS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[COLS_STEP:%.*]], [[COLS_LATCH:%.*]] ] |
| ; CHECK-NEXT: br label [[COLS_BODY:%.*]] |
| ; CHECK: cols.body: |
| ; CHECK-NEXT: br label [[ROWS_HEADER:%.*]] |
| ; CHECK: rows.header: |
| ; CHECK-NEXT: [[ROWS_IV:%.*]] = phi i64 [ 0, [[COLS_BODY]] ], [ [[ROWS_STEP:%.*]], [[ROWS_LATCH:%.*]] ] |
| ; CHECK-NEXT: br label [[ROWS_BODY:%.*]] |
| ; CHECK: rows.body: |
| ; CHECK-NEXT: br label [[INNER_HEADER:%.*]] |
| ; CHECK: inner.header: |
| ; CHECK-NEXT: [[INNER_IV:%.*]] = phi i64 [ 0, [[ROWS_BODY]] ], [ [[INNER_STEP:%.*]], [[INNER_LATCH:%.*]] ] |
| ; CHECK-NEXT: [[RESULT_VEC_0:%.*]] = phi <2 x double> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP11:%.*]], [[INNER_LATCH]] ] |
| ; CHECK-NEXT: [[RESULT_VEC_1:%.*]] = phi <2 x double> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP17:%.*]], [[INNER_LATCH]] ] |
| ; CHECK-NEXT: br label [[INNER_BODY:%.*]] |
| ; CHECK: inner.body: |
| ; CHECK-NEXT: [[TMP0:%.*]] = mul i64 [[INNER_IV]], 2 |
| ; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[TMP0]], [[ROWS_IV]] |
| ; CHECK-NEXT: [[TMP2:%.*]] = getelementptr double, ptr [[A:%.*]], i64 [[TMP1]] |
| ; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, ptr [[TMP2]], align 8 |
| ; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, ptr [[TMP2]], i64 2 |
| ; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x double>, ptr [[VEC_GEP]], align 8 |
| ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[COLS_IV]], 2 |
| ; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[TMP3]], [[INNER_IV]] |
| ; CHECK-NEXT: [[TMP5:%.*]] = getelementptr double, ptr [[B:%.*]], i64 [[TMP4]] |
| ; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x double>, ptr [[TMP5]], align 8 |
| ; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr double, ptr [[TMP5]], i64 2 |
| ; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x double>, ptr [[VEC_GEP3]], align 8 |
| ; CHECK-NEXT: [[BLOCK:%.*]] = shufflevector <2 x double> [[RESULT_VEC_0]], <2 x double> poison, <2 x i32> <i32 0, i32 1> |
| ; CHECK-NEXT: [[BLOCK5:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> poison, <2 x i32> <i32 0, i32 1> |
| ; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x double> [[COL_LOAD2]], i64 0 |
| ; CHECK-NEXT: [[SPLAT_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[TMP6]], i64 0 |
| ; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT]], <2 x double> poison, <2 x i32> zeroinitializer |
| ; CHECK-NEXT: [[TMP7:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK5]], <2 x double> [[SPLAT_SPLAT]], <2 x double> [[BLOCK]]) |
| ; CHECK-NEXT: [[BLOCK6:%.*]] = shufflevector <2 x double> [[COL_LOAD1]], <2 x double> poison, <2 x i32> <i32 0, i32 1> |
| ; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x double> [[COL_LOAD2]], i64 1 |
| ; CHECK-NEXT: [[SPLAT_SPLATINSERT7:%.*]] = insertelement <2 x double> poison, double [[TMP8]], i64 0 |
| ; CHECK-NEXT: [[SPLAT_SPLAT8:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT7]], <2 x double> poison, <2 x i32> zeroinitializer |
| ; CHECK-NEXT: [[TMP9:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK6]], <2 x double> [[SPLAT_SPLAT8]], <2 x double> [[TMP7]]) |
| ; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <2 x double> [[TMP9]], <2 x double> poison, <2 x i32> <i32 0, i32 1> |
| ; CHECK-NEXT: [[TMP11]] = shufflevector <2 x double> [[RESULT_VEC_0]], <2 x double> [[TMP10]], <2 x i32> <i32 2, i32 3> |
| ; CHECK-NEXT: [[BLOCK9:%.*]] = shufflevector <2 x double> [[RESULT_VEC_1]], <2 x double> poison, <2 x i32> <i32 0, i32 1> |
| ; CHECK-NEXT: [[BLOCK10:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> poison, <2 x i32> <i32 0, i32 1> |
| ; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x double> [[COL_LOAD4]], i64 0 |
| ; CHECK-NEXT: [[SPLAT_SPLATINSERT11:%.*]] = insertelement <2 x double> poison, double [[TMP12]], i64 0 |
| ; CHECK-NEXT: [[SPLAT_SPLAT12:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT11]], <2 x double> poison, <2 x i32> zeroinitializer |
| ; CHECK-NEXT: [[TMP13:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK10]], <2 x double> [[SPLAT_SPLAT12]], <2 x double> [[BLOCK9]]) |
| ; CHECK-NEXT: [[BLOCK13:%.*]] = shufflevector <2 x double> [[COL_LOAD1]], <2 x double> poison, <2 x i32> <i32 0, i32 1> |
| ; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x double> [[COL_LOAD4]], i64 1 |
| ; CHECK-NEXT: [[SPLAT_SPLATINSERT14:%.*]] = insertelement <2 x double> poison, double [[TMP14]], i64 0 |
| ; CHECK-NEXT: [[SPLAT_SPLAT15:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT14]], <2 x double> poison, <2 x i32> zeroinitializer |
| ; CHECK-NEXT: [[TMP15:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK13]], <2 x double> [[SPLAT_SPLAT15]], <2 x double> [[TMP13]]) |
| ; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <2 x double> [[TMP15]], <2 x double> poison, <2 x i32> <i32 0, i32 1> |
| ; CHECK-NEXT: [[TMP17]] = shufflevector <2 x double> [[RESULT_VEC_1]], <2 x double> [[TMP16]], <2 x i32> <i32 2, i32 3> |
| ; CHECK-NEXT: br label [[INNER_LATCH]] |
| ; CHECK: inner.latch: |
| ; CHECK-NEXT: [[INNER_STEP]] = add i64 [[INNER_IV]], 2 |
| ; CHECK-NEXT: [[INNER_COND:%.*]] = icmp ne i64 [[INNER_STEP]], 2 |
| ; CHECK-NEXT: br i1 [[INNER_COND]], label [[INNER_HEADER]], label [[ROWS_LATCH]], !llvm.loop [[LOOP3:![0-9]+]] |
| ; CHECK: rows.latch: |
| ; CHECK-NEXT: [[ROWS_STEP]] = add i64 [[ROWS_IV]], 2 |
| ; CHECK-NEXT: [[ROWS_COND:%.*]] = icmp ne i64 [[ROWS_STEP]], 2 |
| ; CHECK-NEXT: [[TMP18:%.*]] = mul i64 [[COLS_IV]], 2 |
| ; CHECK-NEXT: [[TMP19:%.*]] = add i64 [[TMP18]], [[ROWS_IV]] |
| ; CHECK-NEXT: [[TMP20:%.*]] = getelementptr double, ptr [[C:%.*]], i64 [[TMP19]] |
| ; CHECK-NEXT: store <2 x double> [[TMP11]], ptr [[TMP20]], align 8 |
| ; CHECK-NEXT: [[VEC_GEP16:%.*]] = getelementptr double, ptr [[TMP20]], i64 2 |
| ; CHECK-NEXT: store <2 x double> [[TMP17]], ptr [[VEC_GEP16]], align 8 |
| ; CHECK-NEXT: br i1 [[ROWS_COND]], label [[ROWS_HEADER]], label [[COLS_LATCH]] |
| ; CHECK: cols.latch: |
| ; CHECK-NEXT: [[COLS_STEP]] = add i64 [[COLS_IV]], 2 |
| ; CHECK-NEXT: [[COLS_COND:%.*]] = icmp ne i64 [[COLS_STEP]], 2 |
| ; CHECK-NEXT: br i1 [[COLS_COND]], label [[COLS_HEADER]], label [[CONTINUE:%.*]] |
| ; CHECK: continue: |
| ; CHECK-NEXT: ret void |
| ; |
| |
| |
| entry: |
| %a = load <4 x double>, ptr %A, align 8 |
| %b = load volatile <4 x double>, ptr %B, align 8 |
| |
| %c = call <4 x double> @llvm.matrix.multiply(<4 x double> %a, <4 x double> %b, i32 2, i32 2, i32 2) |
| |
| store <4 x double> %c, ptr %C, align 8 |
| ret void |
| } |
| |
| define void @multiply_store_volatile(ptr noalias %A, ptr noalias %B, ptr noalias %C) { |
| ; CHECK-LABEL: @multiply_store_volatile( |
| ; CHECK-NEXT: entry: |
| ; CHECK-NEXT: br label [[COLS_HEADER:%.*]] |
| ; CHECK: cols.header: |
| ; CHECK-NEXT: [[COLS_IV:%.*]] = phi i64 [ 0, [[ENTRY:%.*]] ], [ [[COLS_STEP:%.*]], [[COLS_LATCH:%.*]] ] |
| ; CHECK-NEXT: br label [[COLS_BODY:%.*]] |
| ; CHECK: cols.body: |
| ; CHECK-NEXT: br label [[ROWS_HEADER:%.*]] |
| ; CHECK: rows.header: |
| ; CHECK-NEXT: [[ROWS_IV:%.*]] = phi i64 [ 0, [[COLS_BODY]] ], [ [[ROWS_STEP:%.*]], [[ROWS_LATCH:%.*]] ] |
| ; CHECK-NEXT: br label [[ROWS_BODY:%.*]] |
| ; CHECK: rows.body: |
| ; CHECK-NEXT: br label [[INNER_HEADER:%.*]] |
| ; CHECK: inner.header: |
| ; CHECK-NEXT: [[INNER_IV:%.*]] = phi i64 [ 0, [[ROWS_BODY]] ], [ [[INNER_STEP:%.*]], [[INNER_LATCH:%.*]] ] |
| ; CHECK-NEXT: [[RESULT_VEC_0:%.*]] = phi <2 x double> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP11:%.*]], [[INNER_LATCH]] ] |
| ; CHECK-NEXT: [[RESULT_VEC_1:%.*]] = phi <2 x double> [ zeroinitializer, [[ROWS_BODY]] ], [ [[TMP17:%.*]], [[INNER_LATCH]] ] |
| ; CHECK-NEXT: br label [[INNER_BODY:%.*]] |
| ; CHECK: inner.body: |
| ; CHECK-NEXT: [[TMP0:%.*]] = mul i64 [[INNER_IV]], 2 |
| ; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[TMP0]], [[ROWS_IV]] |
| ; CHECK-NEXT: [[TMP2:%.*]] = getelementptr double, ptr [[A:%.*]], i64 [[TMP1]] |
| ; CHECK-NEXT: [[COL_LOAD:%.*]] = load <2 x double>, ptr [[TMP2]], align 8 |
| ; CHECK-NEXT: [[VEC_GEP:%.*]] = getelementptr double, ptr [[TMP2]], i64 2 |
| ; CHECK-NEXT: [[COL_LOAD1:%.*]] = load <2 x double>, ptr [[VEC_GEP]], align 8 |
| ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[COLS_IV]], 2 |
| ; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[TMP3]], [[INNER_IV]] |
| ; CHECK-NEXT: [[TMP5:%.*]] = getelementptr double, ptr [[B:%.*]], i64 [[TMP4]] |
| ; CHECK-NEXT: [[COL_LOAD2:%.*]] = load <2 x double>, ptr [[TMP5]], align 8 |
| ; CHECK-NEXT: [[VEC_GEP3:%.*]] = getelementptr double, ptr [[TMP5]], i64 2 |
| ; CHECK-NEXT: [[COL_LOAD4:%.*]] = load <2 x double>, ptr [[VEC_GEP3]], align 8 |
| ; CHECK-NEXT: [[BLOCK:%.*]] = shufflevector <2 x double> [[RESULT_VEC_0]], <2 x double> poison, <2 x i32> <i32 0, i32 1> |
| ; CHECK-NEXT: [[BLOCK5:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> poison, <2 x i32> <i32 0, i32 1> |
| ; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x double> [[COL_LOAD2]], i64 0 |
| ; CHECK-NEXT: [[SPLAT_SPLATINSERT:%.*]] = insertelement <2 x double> poison, double [[TMP6]], i64 0 |
| ; CHECK-NEXT: [[SPLAT_SPLAT:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT]], <2 x double> poison, <2 x i32> zeroinitializer |
| ; CHECK-NEXT: [[TMP7:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK5]], <2 x double> [[SPLAT_SPLAT]], <2 x double> [[BLOCK]]) |
| ; CHECK-NEXT: [[BLOCK6:%.*]] = shufflevector <2 x double> [[COL_LOAD1]], <2 x double> poison, <2 x i32> <i32 0, i32 1> |
| ; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x double> [[COL_LOAD2]], i64 1 |
| ; CHECK-NEXT: [[SPLAT_SPLATINSERT7:%.*]] = insertelement <2 x double> poison, double [[TMP8]], i64 0 |
| ; CHECK-NEXT: [[SPLAT_SPLAT8:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT7]], <2 x double> poison, <2 x i32> zeroinitializer |
| ; CHECK-NEXT: [[TMP9:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK6]], <2 x double> [[SPLAT_SPLAT8]], <2 x double> [[TMP7]]) |
| ; CHECK-NEXT: [[TMP10:%.*]] = shufflevector <2 x double> [[TMP9]], <2 x double> poison, <2 x i32> <i32 0, i32 1> |
| ; CHECK-NEXT: [[TMP11]] = shufflevector <2 x double> [[RESULT_VEC_0]], <2 x double> [[TMP10]], <2 x i32> <i32 2, i32 3> |
| ; CHECK-NEXT: [[BLOCK9:%.*]] = shufflevector <2 x double> [[RESULT_VEC_1]], <2 x double> poison, <2 x i32> <i32 0, i32 1> |
| ; CHECK-NEXT: [[BLOCK10:%.*]] = shufflevector <2 x double> [[COL_LOAD]], <2 x double> poison, <2 x i32> <i32 0, i32 1> |
| ; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x double> [[COL_LOAD4]], i64 0 |
| ; CHECK-NEXT: [[SPLAT_SPLATINSERT11:%.*]] = insertelement <2 x double> poison, double [[TMP12]], i64 0 |
| ; CHECK-NEXT: [[SPLAT_SPLAT12:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT11]], <2 x double> poison, <2 x i32> zeroinitializer |
| ; CHECK-NEXT: [[TMP13:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK10]], <2 x double> [[SPLAT_SPLAT12]], <2 x double> [[BLOCK9]]) |
| ; CHECK-NEXT: [[BLOCK13:%.*]] = shufflevector <2 x double> [[COL_LOAD1]], <2 x double> poison, <2 x i32> <i32 0, i32 1> |
| ; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x double> [[COL_LOAD4]], i64 1 |
| ; CHECK-NEXT: [[SPLAT_SPLATINSERT14:%.*]] = insertelement <2 x double> poison, double [[TMP14]], i64 0 |
| ; CHECK-NEXT: [[SPLAT_SPLAT15:%.*]] = shufflevector <2 x double> [[SPLAT_SPLATINSERT14]], <2 x double> poison, <2 x i32> zeroinitializer |
| ; CHECK-NEXT: [[TMP15:%.*]] = call contract <2 x double> @llvm.fmuladd.v2f64(<2 x double> [[BLOCK13]], <2 x double> [[SPLAT_SPLAT15]], <2 x double> [[TMP13]]) |
| ; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <2 x double> [[TMP15]], <2 x double> poison, <2 x i32> <i32 0, i32 1> |
| ; CHECK-NEXT: [[TMP17]] = shufflevector <2 x double> [[RESULT_VEC_1]], <2 x double> [[TMP16]], <2 x i32> <i32 2, i32 3> |
| ; CHECK-NEXT: br label [[INNER_LATCH]] |
| ; CHECK: inner.latch: |
| ; CHECK-NEXT: [[INNER_STEP]] = add i64 [[INNER_IV]], 2 |
| ; CHECK-NEXT: [[INNER_COND:%.*]] = icmp ne i64 [[INNER_STEP]], 2 |
| ; CHECK-NEXT: br i1 [[INNER_COND]], label [[INNER_HEADER]], label [[ROWS_LATCH]], !llvm.loop [[LOOP4:![0-9]+]] |
| ; CHECK: rows.latch: |
| ; CHECK-NEXT: [[ROWS_STEP]] = add i64 [[ROWS_IV]], 2 |
| ; CHECK-NEXT: [[ROWS_COND:%.*]] = icmp ne i64 [[ROWS_STEP]], 2 |
| ; CHECK-NEXT: [[TMP18:%.*]] = mul i64 [[COLS_IV]], 2 |
| ; CHECK-NEXT: [[TMP19:%.*]] = add i64 [[TMP18]], [[ROWS_IV]] |
| ; CHECK-NEXT: [[TMP20:%.*]] = getelementptr double, ptr [[C:%.*]], i64 [[TMP19]] |
| ; CHECK-NEXT: store volatile <2 x double> [[TMP11]], ptr [[TMP20]], align 8 |
| ; CHECK-NEXT: [[VEC_GEP16:%.*]] = getelementptr double, ptr [[TMP20]], i64 2 |
| ; CHECK-NEXT: store volatile <2 x double> [[TMP17]], ptr [[VEC_GEP16]], align 8 |
| ; CHECK-NEXT: br i1 [[ROWS_COND]], label [[ROWS_HEADER]], label [[COLS_LATCH]] |
| ; CHECK: cols.latch: |
| ; CHECK-NEXT: [[COLS_STEP]] = add i64 [[COLS_IV]], 2 |
| ; CHECK-NEXT: [[COLS_COND:%.*]] = icmp ne i64 [[COLS_STEP]], 2 |
| ; CHECK-NEXT: br i1 [[COLS_COND]], label [[COLS_HEADER]], label [[CONTINUE:%.*]] |
| ; CHECK: continue: |
| ; CHECK-NEXT: ret void |
| ; |
| |
| entry: |
| %a = load <4 x double>, ptr %A, align 8 |
| %b = load <4 x double>, ptr %B, align 8 |
| |
| %c = call <4 x double> @llvm.matrix.multiply(<4 x double> %a, <4 x double> %b, i32 2, i32 2, i32 2) |
| |
| store volatile <4 x double> %c, ptr %C, align 8 |
| ret void |
| } |
| |
| declare <4 x double> @llvm.matrix.multiply(<4 x double>, <4 x double>, i32, i32, i32) |
