; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
; Test basic address sanitizer instrumentation.
;
; RUN: opt < %s -passes=hwasan -S | FileCheck %s
; RUN: opt < %s -passes=hwasan -hwasan-inline-fast-path-checks=0 -S | FileCheck %s --check-prefixes=NOFASTPATH
; RUN: opt < %s -passes=hwasan -hwasan-inline-fast-path-checks=1 -S | FileCheck %s --check-prefixes=FASTPATH
; RUN: opt < %s -passes=hwasan -hwasan-recover=0 -hwasan-mapping-offset-dynamic=ifunc -S | FileCheck %s --check-prefixes=ABORT-DYNAMIC-SHADOW
; RUN: opt < %s -passes=hwasan -hwasan-recover=1 -hwasan-mapping-offset-dynamic=ifunc -S | FileCheck %s --check-prefixes=RECOVER-DYNAMIC-SHADOW
; RUN: opt < %s -passes=hwasan -hwasan-recover=0 -hwasan-mapping-offset=0 -S | FileCheck %s --check-prefixes=ABORT-ZERO-BASED-SHADOW
; RUN: opt < %s -passes=hwasan -hwasan-recover=1 -hwasan-mapping-offset=0 -S | FileCheck %s --check-prefixes=RECOVER-ZERO-BASED-SHADOW


target datalayout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128"
target triple = "aarch64--linux-android10000"

define i8 @test_load8(ptr %a) sanitize_hwaddress {
; CHECK-LABEL: define i8 @test_load8
; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT:  entry:
; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 0)
; CHECK-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
; CHECK-NEXT:    ret i8 [[B]]
;
; NOFASTPATH-LABEL: define i8 @test_load8
; NOFASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
; NOFASTPATH-NEXT:  entry:
; NOFASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 0)
; NOFASTPATH-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
; NOFASTPATH-NEXT:    ret i8 [[B]]
;
; FASTPATH-LABEL: define i8 @test_load8
; FASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
; FASTPATH-NEXT:  entry:
; FASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; FASTPATH-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; FASTPATH-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
; FASTPATH-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
; FASTPATH-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
; FASTPATH-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
; FASTPATH-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
; FASTPATH-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
; FASTPATH-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
; FASTPATH-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF2:![0-9]+]]
; FASTPATH:       8:
; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 0)
; FASTPATH-NEXT:    br label [[TMP9]]
; FASTPATH:       9:
; FASTPATH-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
; FASTPATH-NEXT:    ret i8 [[B]]
;
; ABORT-DYNAMIC-SHADOW-LABEL: define i8 @test_load8
; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 0)
; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
; ABORT-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
;
; RECOVER-DYNAMIC-SHADOW-LABEL: define i8 @test_load8
; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF2:![0-9]+]]
; RECOVER-DYNAMIC-SHADOW:       8:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF2]]
; RECOVER-DYNAMIC-SHADOW:       10:
; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "brk #2336", "{x0}"(i64 [[TMP0]])
; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
; RECOVER-DYNAMIC-SHADOW:       11:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 0
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF2]]
; RECOVER-DYNAMIC-SHADOW:       16:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF2]]
; RECOVER-DYNAMIC-SHADOW:       21:
; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
; RECOVER-DYNAMIC-SHADOW:       22:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
;
; ABORT-ZERO-BASED-SHADOW-LABEL: define i8 @test_load8
; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules.fixedshadow(ptr [[A]], i32 0, i64 0)
; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i8 [[B]]
;
; RECOVER-ZERO-BASED-SHADOW-LABEL: define i8 @test_load8
; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0:[0-9]+]] {
; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF2:![0-9]+]]
; RECOVER-ZERO-BASED-SHADOW:       8:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF2]]
; RECOVER-ZERO-BASED-SHADOW:       10:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "brk #2336", "{x0}"(i64 [[TMP0]])
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
; RECOVER-ZERO-BASED-SHADOW:       11:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 0
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF2]]
; RECOVER-ZERO-BASED-SHADOW:       16:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF2]]
; RECOVER-ZERO-BASED-SHADOW:       21:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
; RECOVER-ZERO-BASED-SHADOW:       22:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i8 [[B]]
;
entry:
  %b = load i8, ptr %a, align 4
  ret i8 %b
}

define i16 @test_load16(ptr %a) sanitize_hwaddress {
; CHECK-LABEL: define i16 @test_load16
; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  entry:
; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 1)
; CHECK-NEXT:    [[B:%.*]] = load i16, ptr [[A]], align 4
; CHECK-NEXT:    ret i16 [[B]]
;
; NOFASTPATH-LABEL: define i16 @test_load16
; NOFASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
; NOFASTPATH-NEXT:  entry:
; NOFASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 1)
; NOFASTPATH-NEXT:    [[B:%.*]] = load i16, ptr [[A]], align 4
; NOFASTPATH-NEXT:    ret i16 [[B]]
;
; FASTPATH-LABEL: define i16 @test_load16
; FASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
; FASTPATH-NEXT:  entry:
; FASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; FASTPATH-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; FASTPATH-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
; FASTPATH-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
; FASTPATH-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
; FASTPATH-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
; FASTPATH-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
; FASTPATH-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
; FASTPATH-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
; FASTPATH-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF2]]
; FASTPATH:       8:
; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 1)
; FASTPATH-NEXT:    br label [[TMP9]]
; FASTPATH:       9:
; FASTPATH-NEXT:    [[B:%.*]] = load i16, ptr [[A]], align 4
; FASTPATH-NEXT:    ret i16 [[B]]
;
; ABORT-DYNAMIC-SHADOW-LABEL: define i16 @test_load16
; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 1)
; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i16, ptr [[A]], align 4
; ABORT-DYNAMIC-SHADOW-NEXT:    ret i16 [[B]]
;
; RECOVER-DYNAMIC-SHADOW-LABEL: define i16 @test_load16
; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF2]]
; RECOVER-DYNAMIC-SHADOW:       8:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF2]]
; RECOVER-DYNAMIC-SHADOW:       10:
; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "brk #2337", "{x0}"(i64 [[TMP0]])
; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
; RECOVER-DYNAMIC-SHADOW:       11:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 1
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF2]]
; RECOVER-DYNAMIC-SHADOW:       16:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF2]]
; RECOVER-DYNAMIC-SHADOW:       21:
; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
; RECOVER-DYNAMIC-SHADOW:       22:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i16, ptr [[A]], align 4
; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i16 [[B]]
;
; ABORT-ZERO-BASED-SHADOW-LABEL: define i16 @test_load16
; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules.fixedshadow(ptr [[A]], i32 1, i64 0)
; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i16, ptr [[A]], align 4
; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i16 [[B]]
;
; RECOVER-ZERO-BASED-SHADOW-LABEL: define i16 @test_load16
; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF2]]
; RECOVER-ZERO-BASED-SHADOW:       8:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF2]]
; RECOVER-ZERO-BASED-SHADOW:       10:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "brk #2337", "{x0}"(i64 [[TMP0]])
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
; RECOVER-ZERO-BASED-SHADOW:       11:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 1
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF2]]
; RECOVER-ZERO-BASED-SHADOW:       16:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF2]]
; RECOVER-ZERO-BASED-SHADOW:       21:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
; RECOVER-ZERO-BASED-SHADOW:       22:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i16, ptr [[A]], align 4
; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i16 [[B]]
;
entry:
  %b = load i16, ptr %a, align 4
  ret i16 %b
}

define i32 @test_load32(ptr %a) sanitize_hwaddress {
; CHECK-LABEL: define i32 @test_load32
; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  entry:
; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 2)
; CHECK-NEXT:    [[B:%.*]] = load i32, ptr [[A]], align 4
; CHECK-NEXT:    ret i32 [[B]]
;
; NOFASTPATH-LABEL: define i32 @test_load32
; NOFASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
; NOFASTPATH-NEXT:  entry:
; NOFASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 2)
; NOFASTPATH-NEXT:    [[B:%.*]] = load i32, ptr [[A]], align 4
; NOFASTPATH-NEXT:    ret i32 [[B]]
;
; FASTPATH-LABEL: define i32 @test_load32
; FASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
; FASTPATH-NEXT:  entry:
; FASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; FASTPATH-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; FASTPATH-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
; FASTPATH-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
; FASTPATH-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
; FASTPATH-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
; FASTPATH-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
; FASTPATH-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
; FASTPATH-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
; FASTPATH-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF2]]
; FASTPATH:       8:
; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 2)
; FASTPATH-NEXT:    br label [[TMP9]]
; FASTPATH:       9:
; FASTPATH-NEXT:    [[B:%.*]] = load i32, ptr [[A]], align 4
; FASTPATH-NEXT:    ret i32 [[B]]
;
; ABORT-DYNAMIC-SHADOW-LABEL: define i32 @test_load32
; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 2)
; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i32, ptr [[A]], align 4
; ABORT-DYNAMIC-SHADOW-NEXT:    ret i32 [[B]]
;
; RECOVER-DYNAMIC-SHADOW-LABEL: define i32 @test_load32
; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF2]]
; RECOVER-DYNAMIC-SHADOW:       8:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF2]]
; RECOVER-DYNAMIC-SHADOW:       10:
; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "brk #2338", "{x0}"(i64 [[TMP0]])
; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
; RECOVER-DYNAMIC-SHADOW:       11:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 3
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF2]]
; RECOVER-DYNAMIC-SHADOW:       16:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF2]]
; RECOVER-DYNAMIC-SHADOW:       21:
; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
; RECOVER-DYNAMIC-SHADOW:       22:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i32, ptr [[A]], align 4
; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i32 [[B]]
;
; ABORT-ZERO-BASED-SHADOW-LABEL: define i32 @test_load32
; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules.fixedshadow(ptr [[A]], i32 2, i64 0)
; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i32, ptr [[A]], align 4
; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i32 [[B]]
;
; RECOVER-ZERO-BASED-SHADOW-LABEL: define i32 @test_load32
; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF2]]
; RECOVER-ZERO-BASED-SHADOW:       8:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF2]]
; RECOVER-ZERO-BASED-SHADOW:       10:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "brk #2338", "{x0}"(i64 [[TMP0]])
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
; RECOVER-ZERO-BASED-SHADOW:       11:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 3
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF2]]
; RECOVER-ZERO-BASED-SHADOW:       16:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF2]]
; RECOVER-ZERO-BASED-SHADOW:       21:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
; RECOVER-ZERO-BASED-SHADOW:       22:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i32, ptr [[A]], align 4
; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i32 [[B]]
;
entry:
  %b = load i32, ptr %a, align 4
  ret i32 %b
}

define i64 @test_load64(ptr %a) sanitize_hwaddress {
; CHECK-LABEL: define i64 @test_load64
; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  entry:
; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 3)
; CHECK-NEXT:    [[B:%.*]] = load i64, ptr [[A]], align 8
; CHECK-NEXT:    ret i64 [[B]]
;
; NOFASTPATH-LABEL: define i64 @test_load64
; NOFASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
; NOFASTPATH-NEXT:  entry:
; NOFASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 3)
; NOFASTPATH-NEXT:    [[B:%.*]] = load i64, ptr [[A]], align 8
; NOFASTPATH-NEXT:    ret i64 [[B]]
;
; FASTPATH-LABEL: define i64 @test_load64
; FASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
; FASTPATH-NEXT:  entry:
; FASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; FASTPATH-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; FASTPATH-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
; FASTPATH-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
; FASTPATH-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
; FASTPATH-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
; FASTPATH-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
; FASTPATH-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
; FASTPATH-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
; FASTPATH-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF2]]
; FASTPATH:       8:
; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 3)
; FASTPATH-NEXT:    br label [[TMP9]]
; FASTPATH:       9:
; FASTPATH-NEXT:    [[B:%.*]] = load i64, ptr [[A]], align 8
; FASTPATH-NEXT:    ret i64 [[B]]
;
; ABORT-DYNAMIC-SHADOW-LABEL: define i64 @test_load64
; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 3)
; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i64, ptr [[A]], align 8
; ABORT-DYNAMIC-SHADOW-NEXT:    ret i64 [[B]]
;
; RECOVER-DYNAMIC-SHADOW-LABEL: define i64 @test_load64
; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF2]]
; RECOVER-DYNAMIC-SHADOW:       8:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF2]]
; RECOVER-DYNAMIC-SHADOW:       10:
; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "brk #2339", "{x0}"(i64 [[TMP0]])
; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
; RECOVER-DYNAMIC-SHADOW:       11:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 7
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF2]]
; RECOVER-DYNAMIC-SHADOW:       16:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF2]]
; RECOVER-DYNAMIC-SHADOW:       21:
; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
; RECOVER-DYNAMIC-SHADOW:       22:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i64, ptr [[A]], align 8
; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i64 [[B]]
;
; ABORT-ZERO-BASED-SHADOW-LABEL: define i64 @test_load64
; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules.fixedshadow(ptr [[A]], i32 3, i64 0)
; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i64, ptr [[A]], align 8
; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i64 [[B]]
;
; RECOVER-ZERO-BASED-SHADOW-LABEL: define i64 @test_load64
; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF2]]
; RECOVER-ZERO-BASED-SHADOW:       8:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF2]]
; RECOVER-ZERO-BASED-SHADOW:       10:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "brk #2339", "{x0}"(i64 [[TMP0]])
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
; RECOVER-ZERO-BASED-SHADOW:       11:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 7
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF2]]
; RECOVER-ZERO-BASED-SHADOW:       16:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF2]]
; RECOVER-ZERO-BASED-SHADOW:       21:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
; RECOVER-ZERO-BASED-SHADOW:       22:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i64, ptr [[A]], align 8
; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i64 [[B]]
;
entry:
  %b = load i64, ptr %a, align 8
  ret i64 %b
}

define i128 @test_load128(ptr %a) sanitize_hwaddress {
; CHECK-LABEL: define i128 @test_load128
; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  entry:
; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 4)
; CHECK-NEXT:    [[B:%.*]] = load i128, ptr [[A]], align 16
; CHECK-NEXT:    ret i128 [[B]]
;
; NOFASTPATH-LABEL: define i128 @test_load128
; NOFASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
; NOFASTPATH-NEXT:  entry:
; NOFASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 4)
; NOFASTPATH-NEXT:    [[B:%.*]] = load i128, ptr [[A]], align 16
; NOFASTPATH-NEXT:    ret i128 [[B]]
;
; FASTPATH-LABEL: define i128 @test_load128
; FASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
; FASTPATH-NEXT:  entry:
; FASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; FASTPATH-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; FASTPATH-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
; FASTPATH-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
; FASTPATH-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
; FASTPATH-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
; FASTPATH-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
; FASTPATH-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
; FASTPATH-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
; FASTPATH-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF2]]
; FASTPATH:       8:
; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 4)
; FASTPATH-NEXT:    br label [[TMP9]]
; FASTPATH:       9:
; FASTPATH-NEXT:    [[B:%.*]] = load i128, ptr [[A]], align 16
; FASTPATH-NEXT:    ret i128 [[B]]
;
; ABORT-DYNAMIC-SHADOW-LABEL: define i128 @test_load128
; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 4)
; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i128, ptr [[A]], align 16
; ABORT-DYNAMIC-SHADOW-NEXT:    ret i128 [[B]]
;
; RECOVER-DYNAMIC-SHADOW-LABEL: define i128 @test_load128
; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF2]]
; RECOVER-DYNAMIC-SHADOW:       8:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF2]]
; RECOVER-DYNAMIC-SHADOW:       10:
; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "brk #2340", "{x0}"(i64 [[TMP0]])
; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
; RECOVER-DYNAMIC-SHADOW:       11:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 15
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF2]]
; RECOVER-DYNAMIC-SHADOW:       16:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF2]]
; RECOVER-DYNAMIC-SHADOW:       21:
; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
; RECOVER-DYNAMIC-SHADOW:       22:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i128, ptr [[A]], align 16
; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i128 [[B]]
;
; ABORT-ZERO-BASED-SHADOW-LABEL: define i128 @test_load128
; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules.fixedshadow(ptr [[A]], i32 4, i64 0)
; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i128, ptr [[A]], align 16
; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i128 [[B]]
;
; RECOVER-ZERO-BASED-SHADOW-LABEL: define i128 @test_load128
; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF2]]
; RECOVER-ZERO-BASED-SHADOW:       8:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF2]]
; RECOVER-ZERO-BASED-SHADOW:       10:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "brk #2340", "{x0}"(i64 [[TMP0]])
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
; RECOVER-ZERO-BASED-SHADOW:       11:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 15
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF2]]
; RECOVER-ZERO-BASED-SHADOW:       16:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF2]]
; RECOVER-ZERO-BASED-SHADOW:       21:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
; RECOVER-ZERO-BASED-SHADOW:       22:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i128, ptr [[A]], align 16
; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i128 [[B]]
;
entry:
  %b = load i128, ptr %a, align 16
  ret i128 %b
}

define i40 @test_load40(ptr %a) sanitize_hwaddress {
; CHECK-LABEL: define i40 @test_load40
; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  entry:
; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; CHECK-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; CHECK-NEXT:    call void @__hwasan_loadN(i64 [[TMP0]], i64 5)
; CHECK-NEXT:    [[B:%.*]] = load i40, ptr [[A]], align 4
; CHECK-NEXT:    ret i40 [[B]]
;
; NOFASTPATH-LABEL: define i40 @test_load40
; NOFASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
; NOFASTPATH-NEXT:  entry:
; NOFASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; NOFASTPATH-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; NOFASTPATH-NEXT:    call void @__hwasan_loadN(i64 [[TMP0]], i64 5)
; NOFASTPATH-NEXT:    [[B:%.*]] = load i40, ptr [[A]], align 4
; NOFASTPATH-NEXT:    ret i40 [[B]]
;
; FASTPATH-LABEL: define i40 @test_load40
; FASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
; FASTPATH-NEXT:  entry:
; FASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; FASTPATH-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; FASTPATH-NEXT:    call void @__hwasan_loadN(i64 [[TMP0]], i64 5)
; FASTPATH-NEXT:    [[B:%.*]] = load i40, ptr [[A]], align 4
; FASTPATH-NEXT:    ret i40 [[B]]
;
; ABORT-DYNAMIC-SHADOW-LABEL: define i40 @test_load40
; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; ABORT-DYNAMIC-SHADOW-NEXT:    call void @__hwasan_loadN(i64 [[TMP0]], i64 5)
; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i40, ptr [[A]], align 4
; ABORT-DYNAMIC-SHADOW-NEXT:    ret i40 [[B]]
;
; RECOVER-DYNAMIC-SHADOW-LABEL: define i40 @test_load40
; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; RECOVER-DYNAMIC-SHADOW-NEXT:    call void @__hwasan_loadN_noabort(i64 [[TMP0]], i64 5)
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i40, ptr [[A]], align 4
; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i40 [[B]]
;
; ABORT-ZERO-BASED-SHADOW-LABEL: define i40 @test_load40
; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @__hwasan_loadN(i64 [[TMP0]], i64 5)
; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i40, ptr [[A]], align 4
; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i40 [[B]]
;
; RECOVER-ZERO-BASED-SHADOW-LABEL: define i40 @test_load40
; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR0]] {
; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void @__hwasan_loadN_noabort(i64 [[TMP0]], i64 5)
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i40, ptr [[A]], align 4
; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i40 [[B]]
;
entry:
  %b = load i40, ptr %a, align 4
  ret i40 %b
}

define void @test_store8(ptr %a, i8 %b) sanitize_hwaddress {
; CHECK-LABEL: define void @test_store8
; CHECK-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  entry:
; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 16)
; CHECK-NEXT:    store i8 [[B]], ptr [[A]], align 4
; CHECK-NEXT:    ret void
;
; NOFASTPATH-LABEL: define void @test_store8
; NOFASTPATH-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
; NOFASTPATH-NEXT:  entry:
; NOFASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 16)
; NOFASTPATH-NEXT:    store i8 [[B]], ptr [[A]], align 4
; NOFASTPATH-NEXT:    ret void
;
; FASTPATH-LABEL: define void @test_store8
; FASTPATH-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
; FASTPATH-NEXT:  entry:
; FASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; FASTPATH-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; FASTPATH-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
; FASTPATH-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
; FASTPATH-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
; FASTPATH-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
; FASTPATH-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
; FASTPATH-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
; FASTPATH-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
; FASTPATH-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF2]]
; FASTPATH:       8:
; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 16)
; FASTPATH-NEXT:    br label [[TMP9]]
; FASTPATH:       9:
; FASTPATH-NEXT:    store i8 [[B]], ptr [[A]], align 4
; FASTPATH-NEXT:    ret void
;
; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store8
; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 16)
; ABORT-DYNAMIC-SHADOW-NEXT:    store i8 [[B]], ptr [[A]], align 4
; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
;
; RECOVER-DYNAMIC-SHADOW-LABEL: define void @test_store8
; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF2]]
; RECOVER-DYNAMIC-SHADOW:       8:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF2]]
; RECOVER-DYNAMIC-SHADOW:       10:
; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "brk #2352", "{x0}"(i64 [[TMP0]])
; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
; RECOVER-DYNAMIC-SHADOW:       11:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 0
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF2]]
; RECOVER-DYNAMIC-SHADOW:       16:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF2]]
; RECOVER-DYNAMIC-SHADOW:       21:
; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
; RECOVER-DYNAMIC-SHADOW:       22:
; RECOVER-DYNAMIC-SHADOW-NEXT:    store i8 [[B]], ptr [[A]], align 4
; RECOVER-DYNAMIC-SHADOW-NEXT:    ret void
;
; ABORT-ZERO-BASED-SHADOW-LABEL: define void @test_store8
; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules.fixedshadow(ptr [[A]], i32 16, i64 0)
; ABORT-ZERO-BASED-SHADOW-NEXT:    store i8 [[B]], ptr [[A]], align 4
; ABORT-ZERO-BASED-SHADOW-NEXT:    ret void
;
; RECOVER-ZERO-BASED-SHADOW-LABEL: define void @test_store8
; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i8 [[B:%.*]]) #[[ATTR0]] {
; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF2]]
; RECOVER-ZERO-BASED-SHADOW:       8:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF2]]
; RECOVER-ZERO-BASED-SHADOW:       10:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "brk #2352", "{x0}"(i64 [[TMP0]])
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
; RECOVER-ZERO-BASED-SHADOW:       11:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 0
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF2]]
; RECOVER-ZERO-BASED-SHADOW:       16:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF2]]
; RECOVER-ZERO-BASED-SHADOW:       21:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
; RECOVER-ZERO-BASED-SHADOW:       22:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    store i8 [[B]], ptr [[A]], align 4
; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret void
;
entry:
  store i8 %b, ptr %a, align 4
  ret void
}

define void @test_store16(ptr %a, i16 %b) sanitize_hwaddress {
; CHECK-LABEL: define void @test_store16
; CHECK-SAME: (ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  entry:
; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 17)
; CHECK-NEXT:    store i16 [[B]], ptr [[A]], align 4
; CHECK-NEXT:    ret void
;
; NOFASTPATH-LABEL: define void @test_store16
; NOFASTPATH-SAME: (ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
; NOFASTPATH-NEXT:  entry:
; NOFASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 17)
; NOFASTPATH-NEXT:    store i16 [[B]], ptr [[A]], align 4
; NOFASTPATH-NEXT:    ret void
;
; FASTPATH-LABEL: define void @test_store16
; FASTPATH-SAME: (ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
; FASTPATH-NEXT:  entry:
; FASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; FASTPATH-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; FASTPATH-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
; FASTPATH-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
; FASTPATH-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
; FASTPATH-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
; FASTPATH-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
; FASTPATH-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
; FASTPATH-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
; FASTPATH-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF2]]
; FASTPATH:       8:
; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 17)
; FASTPATH-NEXT:    br label [[TMP9]]
; FASTPATH:       9:
; FASTPATH-NEXT:    store i16 [[B]], ptr [[A]], align 4
; FASTPATH-NEXT:    ret void
;
; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store16
; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 17)
; ABORT-DYNAMIC-SHADOW-NEXT:    store i16 [[B]], ptr [[A]], align 4
; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
;
; RECOVER-DYNAMIC-SHADOW-LABEL: define void @test_store16
; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF2]]
; RECOVER-DYNAMIC-SHADOW:       8:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF2]]
; RECOVER-DYNAMIC-SHADOW:       10:
; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "brk #2353", "{x0}"(i64 [[TMP0]])
; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
; RECOVER-DYNAMIC-SHADOW:       11:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 1
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF2]]
; RECOVER-DYNAMIC-SHADOW:       16:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF2]]
; RECOVER-DYNAMIC-SHADOW:       21:
; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
; RECOVER-DYNAMIC-SHADOW:       22:
; RECOVER-DYNAMIC-SHADOW-NEXT:    store i16 [[B]], ptr [[A]], align 4
; RECOVER-DYNAMIC-SHADOW-NEXT:    ret void
;
; ABORT-ZERO-BASED-SHADOW-LABEL: define void @test_store16
; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules.fixedshadow(ptr [[A]], i32 17, i64 0)
; ABORT-ZERO-BASED-SHADOW-NEXT:    store i16 [[B]], ptr [[A]], align 4
; ABORT-ZERO-BASED-SHADOW-NEXT:    ret void
;
; RECOVER-ZERO-BASED-SHADOW-LABEL: define void @test_store16
; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i16 [[B:%.*]]) #[[ATTR0]] {
; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF2]]
; RECOVER-ZERO-BASED-SHADOW:       8:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF2]]
; RECOVER-ZERO-BASED-SHADOW:       10:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "brk #2353", "{x0}"(i64 [[TMP0]])
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
; RECOVER-ZERO-BASED-SHADOW:       11:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 1
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF2]]
; RECOVER-ZERO-BASED-SHADOW:       16:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF2]]
; RECOVER-ZERO-BASED-SHADOW:       21:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
; RECOVER-ZERO-BASED-SHADOW:       22:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    store i16 [[B]], ptr [[A]], align 4
; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret void
;
entry:
  store i16 %b, ptr %a, align 4
  ret void
}

define void @test_store32(ptr %a, i32 %b) sanitize_hwaddress {
; CHECK-LABEL: define void @test_store32
; CHECK-SAME: (ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  entry:
; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 18)
; CHECK-NEXT:    store i32 [[B]], ptr [[A]], align 4
; CHECK-NEXT:    ret void
;
; NOFASTPATH-LABEL: define void @test_store32
; NOFASTPATH-SAME: (ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
; NOFASTPATH-NEXT:  entry:
; NOFASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 18)
; NOFASTPATH-NEXT:    store i32 [[B]], ptr [[A]], align 4
; NOFASTPATH-NEXT:    ret void
;
; FASTPATH-LABEL: define void @test_store32
; FASTPATH-SAME: (ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
; FASTPATH-NEXT:  entry:
; FASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; FASTPATH-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; FASTPATH-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
; FASTPATH-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
; FASTPATH-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
; FASTPATH-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
; FASTPATH-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
; FASTPATH-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
; FASTPATH-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
; FASTPATH-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF2]]
; FASTPATH:       8:
; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 18)
; FASTPATH-NEXT:    br label [[TMP9]]
; FASTPATH:       9:
; FASTPATH-NEXT:    store i32 [[B]], ptr [[A]], align 4
; FASTPATH-NEXT:    ret void
;
; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store32
; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 18)
; ABORT-DYNAMIC-SHADOW-NEXT:    store i32 [[B]], ptr [[A]], align 4
; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
;
; RECOVER-DYNAMIC-SHADOW-LABEL: define void @test_store32
; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF2]]
; RECOVER-DYNAMIC-SHADOW:       8:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF2]]
; RECOVER-DYNAMIC-SHADOW:       10:
; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "brk #2354", "{x0}"(i64 [[TMP0]])
; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
; RECOVER-DYNAMIC-SHADOW:       11:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 3
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF2]]
; RECOVER-DYNAMIC-SHADOW:       16:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF2]]
; RECOVER-DYNAMIC-SHADOW:       21:
; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
; RECOVER-DYNAMIC-SHADOW:       22:
; RECOVER-DYNAMIC-SHADOW-NEXT:    store i32 [[B]], ptr [[A]], align 4
; RECOVER-DYNAMIC-SHADOW-NEXT:    ret void
;
; ABORT-ZERO-BASED-SHADOW-LABEL: define void @test_store32
; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules.fixedshadow(ptr [[A]], i32 18, i64 0)
; ABORT-ZERO-BASED-SHADOW-NEXT:    store i32 [[B]], ptr [[A]], align 4
; ABORT-ZERO-BASED-SHADOW-NEXT:    ret void
;
; RECOVER-ZERO-BASED-SHADOW-LABEL: define void @test_store32
; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i32 [[B:%.*]]) #[[ATTR0]] {
; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF2]]
; RECOVER-ZERO-BASED-SHADOW:       8:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF2]]
; RECOVER-ZERO-BASED-SHADOW:       10:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "brk #2354", "{x0}"(i64 [[TMP0]])
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
; RECOVER-ZERO-BASED-SHADOW:       11:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 3
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF2]]
; RECOVER-ZERO-BASED-SHADOW:       16:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF2]]
; RECOVER-ZERO-BASED-SHADOW:       21:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
; RECOVER-ZERO-BASED-SHADOW:       22:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    store i32 [[B]], ptr [[A]], align 4
; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret void
;
entry:
  store i32 %b, ptr %a, align 4
  ret void
}

define void @test_store64(ptr %a, i64 %b) sanitize_hwaddress {
; CHECK-LABEL: define void @test_store64
; CHECK-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  entry:
; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 19)
; CHECK-NEXT:    store i64 [[B]], ptr [[A]], align 8
; CHECK-NEXT:    ret void
;
; NOFASTPATH-LABEL: define void @test_store64
; NOFASTPATH-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
; NOFASTPATH-NEXT:  entry:
; NOFASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 19)
; NOFASTPATH-NEXT:    store i64 [[B]], ptr [[A]], align 8
; NOFASTPATH-NEXT:    ret void
;
; FASTPATH-LABEL: define void @test_store64
; FASTPATH-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
; FASTPATH-NEXT:  entry:
; FASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; FASTPATH-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; FASTPATH-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
; FASTPATH-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
; FASTPATH-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
; FASTPATH-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
; FASTPATH-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
; FASTPATH-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
; FASTPATH-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
; FASTPATH-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF2]]
; FASTPATH:       8:
; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 19)
; FASTPATH-NEXT:    br label [[TMP9]]
; FASTPATH:       9:
; FASTPATH-NEXT:    store i64 [[B]], ptr [[A]], align 8
; FASTPATH-NEXT:    ret void
;
; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store64
; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 19)
; ABORT-DYNAMIC-SHADOW-NEXT:    store i64 [[B]], ptr [[A]], align 8
; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
;
; RECOVER-DYNAMIC-SHADOW-LABEL: define void @test_store64
; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF2]]
; RECOVER-DYNAMIC-SHADOW:       8:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF2]]
; RECOVER-DYNAMIC-SHADOW:       10:
; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "brk #2355", "{x0}"(i64 [[TMP0]])
; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
; RECOVER-DYNAMIC-SHADOW:       11:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 7
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF2]]
; RECOVER-DYNAMIC-SHADOW:       16:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF2]]
; RECOVER-DYNAMIC-SHADOW:       21:
; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
; RECOVER-DYNAMIC-SHADOW:       22:
; RECOVER-DYNAMIC-SHADOW-NEXT:    store i64 [[B]], ptr [[A]], align 8
; RECOVER-DYNAMIC-SHADOW-NEXT:    ret void
;
; ABORT-ZERO-BASED-SHADOW-LABEL: define void @test_store64
; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules.fixedshadow(ptr [[A]], i32 19, i64 0)
; ABORT-ZERO-BASED-SHADOW-NEXT:    store i64 [[B]], ptr [[A]], align 8
; ABORT-ZERO-BASED-SHADOW-NEXT:    ret void
;
; RECOVER-ZERO-BASED-SHADOW-LABEL: define void @test_store64
; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF2]]
; RECOVER-ZERO-BASED-SHADOW:       8:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF2]]
; RECOVER-ZERO-BASED-SHADOW:       10:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "brk #2355", "{x0}"(i64 [[TMP0]])
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
; RECOVER-ZERO-BASED-SHADOW:       11:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 7
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF2]]
; RECOVER-ZERO-BASED-SHADOW:       16:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF2]]
; RECOVER-ZERO-BASED-SHADOW:       21:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
; RECOVER-ZERO-BASED-SHADOW:       22:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    store i64 [[B]], ptr [[A]], align 8
; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret void
;
entry:
  store i64 %b, ptr %a, align 8
  ret void
}

define void @test_store128(ptr %a, i128 %b) sanitize_hwaddress {
; CHECK-LABEL: define void @test_store128
; CHECK-SAME: (ptr [[A:%.*]], i128 [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  entry:
; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; CHECK-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 20)
; CHECK-NEXT:    store i128 [[B]], ptr [[A]], align 16
; CHECK-NEXT:    ret void
;
; NOFASTPATH-LABEL: define void @test_store128
; NOFASTPATH-SAME: (ptr [[A:%.*]], i128 [[B:%.*]]) #[[ATTR0]] {
; NOFASTPATH-NEXT:  entry:
; NOFASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; NOFASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 20)
; NOFASTPATH-NEXT:    store i128 [[B]], ptr [[A]], align 16
; NOFASTPATH-NEXT:    ret void
;
; FASTPATH-LABEL: define void @test_store128
; FASTPATH-SAME: (ptr [[A:%.*]], i128 [[B:%.*]]) #[[ATTR0]] {
; FASTPATH-NEXT:  entry:
; FASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; FASTPATH-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; FASTPATH-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
; FASTPATH-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
; FASTPATH-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
; FASTPATH-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
; FASTPATH-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
; FASTPATH-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
; FASTPATH-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
; FASTPATH-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP9:%.*]], !prof [[PROF2]]
; FASTPATH:       8:
; FASTPATH-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 20)
; FASTPATH-NEXT:    br label [[TMP9]]
; FASTPATH:       9:
; FASTPATH-NEXT:    store i128 [[B]], ptr [[A]], align 16
; FASTPATH-NEXT:    ret void
;
; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store128
; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i128 [[B:%.*]]) #[[ATTR0]] {
; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; ABORT-DYNAMIC-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules(ptr [[DOTHWASAN_SHADOW]], ptr [[A]], i32 20)
; ABORT-DYNAMIC-SHADOW-NEXT:    store i128 [[B]], ptr [[A]], align 16
; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
;
; RECOVER-DYNAMIC-SHADOW-LABEL: define void @test_store128
; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i128 [[B:%.*]]) #[[ATTR0]] {
; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP5:%.*]] = getelementptr i8, ptr [[DOTHWASAN_SHADOW]], i64 [[TMP4]]
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF2]]
; RECOVER-DYNAMIC-SHADOW:       8:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF2]]
; RECOVER-DYNAMIC-SHADOW:       10:
; RECOVER-DYNAMIC-SHADOW-NEXT:    call void asm sideeffect "brk #2356", "{x0}"(i64 [[TMP0]])
; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP21:%.*]]
; RECOVER-DYNAMIC-SHADOW:       11:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 15
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF2]]
; RECOVER-DYNAMIC-SHADOW:       16:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
; RECOVER-DYNAMIC-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF2]]
; RECOVER-DYNAMIC-SHADOW:       21:
; RECOVER-DYNAMIC-SHADOW-NEXT:    br label [[TMP22]]
; RECOVER-DYNAMIC-SHADOW:       22:
; RECOVER-DYNAMIC-SHADOW-NEXT:    store i128 [[B]], ptr [[A]], align 16
; RECOVER-DYNAMIC-SHADOW-NEXT:    ret void
;
; ABORT-ZERO-BASED-SHADOW-LABEL: define void @test_store128
; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i128 [[B:%.*]]) #[[ATTR0]] {
; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @llvm.hwasan.check.memaccess.shortgranules.fixedshadow(ptr [[A]], i32 20, i64 0)
; ABORT-ZERO-BASED-SHADOW-NEXT:    store i128 [[B]], ptr [[A]], align 16
; ABORT-ZERO-BASED-SHADOW-NEXT:    ret void
;
; RECOVER-ZERO-BASED-SHADOW-LABEL: define void @test_store128
; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i128 [[B:%.*]]) #[[ATTR0]] {
; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP1:%.*]] = lshr i64 [[TMP0]], 56
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP2:%.*]] = trunc i64 [[TMP1]] to i8
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP3:%.*]] = and i64 [[TMP0]], 72057594037927935
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP4:%.*]] = lshr i64 [[TMP3]], 4
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP5:%.*]] = inttoptr i64 [[TMP4]] to ptr
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP6:%.*]] = load i8, ptr [[TMP5]], align 1
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP7:%.*]] = icmp ne i8 [[TMP2]], [[TMP6]]
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP7]], label [[TMP8:%.*]], label [[TMP22:%.*]], !prof [[PROF2]]
; RECOVER-ZERO-BASED-SHADOW:       8:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP9:%.*]] = icmp ugt i8 [[TMP6]], 15
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP9]], label [[TMP10:%.*]], label [[TMP11:%.*]], !prof [[PROF2]]
; RECOVER-ZERO-BASED-SHADOW:       10:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void asm sideeffect "brk #2356", "{x0}"(i64 [[TMP0]])
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP21:%.*]]
; RECOVER-ZERO-BASED-SHADOW:       11:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP12:%.*]] = and i64 [[TMP0]], 15
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP13:%.*]] = trunc i64 [[TMP12]] to i8
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP14:%.*]] = add i8 [[TMP13]], 15
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP15:%.*]] = icmp uge i8 [[TMP14]], [[TMP6]]
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP15]], label [[TMP10]], label [[TMP16:%.*]], !prof [[PROF2]]
; RECOVER-ZERO-BASED-SHADOW:       16:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP17:%.*]] = or i64 [[TMP3]], 15
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP18:%.*]] = inttoptr i64 [[TMP17]] to ptr
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP19:%.*]] = load i8, ptr [[TMP18]], align 1
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP20:%.*]] = icmp ne i8 [[TMP2]], [[TMP19]]
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br i1 [[TMP20]], label [[TMP10]], label [[TMP21]], !prof [[PROF2]]
; RECOVER-ZERO-BASED-SHADOW:       21:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    br label [[TMP22]]
; RECOVER-ZERO-BASED-SHADOW:       22:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    store i128 [[B]], ptr [[A]], align 16
; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret void
;
entry:
  store i128 %b, ptr %a, align 16
  ret void
}

define void @test_store40(ptr %a, i40 %b) sanitize_hwaddress {
; CHECK-LABEL: define void @test_store40
; CHECK-SAME: (ptr [[A:%.*]], i40 [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  entry:
; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; CHECK-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; CHECK-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 5)
; CHECK-NEXT:    store i40 [[B]], ptr [[A]], align 4
; CHECK-NEXT:    ret void
;
; NOFASTPATH-LABEL: define void @test_store40
; NOFASTPATH-SAME: (ptr [[A:%.*]], i40 [[B:%.*]]) #[[ATTR0]] {
; NOFASTPATH-NEXT:  entry:
; NOFASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; NOFASTPATH-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; NOFASTPATH-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 5)
; NOFASTPATH-NEXT:    store i40 [[B]], ptr [[A]], align 4
; NOFASTPATH-NEXT:    ret void
;
; FASTPATH-LABEL: define void @test_store40
; FASTPATH-SAME: (ptr [[A:%.*]], i40 [[B:%.*]]) #[[ATTR0]] {
; FASTPATH-NEXT:  entry:
; FASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; FASTPATH-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; FASTPATH-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 5)
; FASTPATH-NEXT:    store i40 [[B]], ptr [[A]], align 4
; FASTPATH-NEXT:    ret void
;
; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store40
; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i40 [[B:%.*]]) #[[ATTR0]] {
; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; ABORT-DYNAMIC-SHADOW-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 5)
; ABORT-DYNAMIC-SHADOW-NEXT:    store i40 [[B]], ptr [[A]], align 4
; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
;
; RECOVER-DYNAMIC-SHADOW-LABEL: define void @test_store40
; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i40 [[B:%.*]]) #[[ATTR0]] {
; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; RECOVER-DYNAMIC-SHADOW-NEXT:    call void @__hwasan_storeN_noabort(i64 [[TMP0]], i64 5)
; RECOVER-DYNAMIC-SHADOW-NEXT:    store i40 [[B]], ptr [[A]], align 4
; RECOVER-DYNAMIC-SHADOW-NEXT:    ret void
;
; ABORT-ZERO-BASED-SHADOW-LABEL: define void @test_store40
; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i40 [[B:%.*]]) #[[ATTR0]] {
; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 5)
; ABORT-ZERO-BASED-SHADOW-NEXT:    store i40 [[B]], ptr [[A]], align 4
; ABORT-ZERO-BASED-SHADOW-NEXT:    ret void
;
; RECOVER-ZERO-BASED-SHADOW-LABEL: define void @test_store40
; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i40 [[B:%.*]]) #[[ATTR0]] {
; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void @__hwasan_storeN_noabort(i64 [[TMP0]], i64 5)
; RECOVER-ZERO-BASED-SHADOW-NEXT:    store i40 [[B]], ptr [[A]], align 4
; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret void
;
entry:
  store i40 %b, ptr %a, align 4
  ret void
}

define void @test_store_unaligned(ptr %a, i64 %b) sanitize_hwaddress {
; CHECK-LABEL: define void @test_store_unaligned
; CHECK-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  entry:
; CHECK-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; CHECK-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; CHECK-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 8)
; CHECK-NEXT:    store i64 [[B]], ptr [[A]], align 4
; CHECK-NEXT:    ret void
;
; NOFASTPATH-LABEL: define void @test_store_unaligned
; NOFASTPATH-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
; NOFASTPATH-NEXT:  entry:
; NOFASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; NOFASTPATH-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; NOFASTPATH-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 8)
; NOFASTPATH-NEXT:    store i64 [[B]], ptr [[A]], align 4
; NOFASTPATH-NEXT:    ret void
;
; FASTPATH-LABEL: define void @test_store_unaligned
; FASTPATH-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
; FASTPATH-NEXT:  entry:
; FASTPATH-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; FASTPATH-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; FASTPATH-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 8)
; FASTPATH-NEXT:    store i64 [[B]], ptr [[A]], align 4
; FASTPATH-NEXT:    ret void
;
; ABORT-DYNAMIC-SHADOW-LABEL: define void @test_store_unaligned
; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
; ABORT-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; ABORT-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; ABORT-DYNAMIC-SHADOW-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 8)
; ABORT-DYNAMIC-SHADOW-NEXT:    store i64 [[B]], ptr [[A]], align 4
; ABORT-DYNAMIC-SHADOW-NEXT:    ret void
;
; RECOVER-DYNAMIC-SHADOW-LABEL: define void @test_store_unaligned
; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr @__hwasan_shadow)
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; RECOVER-DYNAMIC-SHADOW-NEXT:    call void @__hwasan_storeN_noabort(i64 [[TMP0]], i64 8)
; RECOVER-DYNAMIC-SHADOW-NEXT:    store i64 [[B]], ptr [[A]], align 4
; RECOVER-DYNAMIC-SHADOW-NEXT:    ret void
;
; ABORT-ZERO-BASED-SHADOW-LABEL: define void @test_store_unaligned
; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
; ABORT-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
; ABORT-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; ABORT-ZERO-BASED-SHADOW-NEXT:    call void @__hwasan_storeN(i64 [[TMP0]], i64 8)
; ABORT-ZERO-BASED-SHADOW-NEXT:    store i64 [[B]], ptr [[A]], align 4
; ABORT-ZERO-BASED-SHADOW-NEXT:    ret void
;
; RECOVER-ZERO-BASED-SHADOW-LABEL: define void @test_store_unaligned
; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]], i64 [[B:%.*]]) #[[ATTR0]] {
; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[DOTHWASAN_SHADOW:%.*]] = call ptr asm "", "=r,0"(ptr null)
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[TMP0:%.*]] = ptrtoint ptr [[A]] to i64
; RECOVER-ZERO-BASED-SHADOW-NEXT:    call void @__hwasan_storeN_noabort(i64 [[TMP0]], i64 8)
; RECOVER-ZERO-BASED-SHADOW-NEXT:    store i64 [[B]], ptr [[A]], align 4
; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret void
;
entry:
  store i64 %b, ptr %a, align 4
  ret void
}

define i8 @test_load_noattr(ptr %a) {
; CHECK-LABEL: define i8 @test_load_noattr
; CHECK-SAME: (ptr [[A:%.*]]) {
; CHECK-NEXT:  entry:
; CHECK-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
; CHECK-NEXT:    ret i8 [[B]]
;
; NOFASTPATH-LABEL: define i8 @test_load_noattr
; NOFASTPATH-SAME: (ptr [[A:%.*]]) {
; NOFASTPATH-NEXT:  entry:
; NOFASTPATH-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
; NOFASTPATH-NEXT:    ret i8 [[B]]
;
; FASTPATH-LABEL: define i8 @test_load_noattr
; FASTPATH-SAME: (ptr [[A:%.*]]) {
; FASTPATH-NEXT:  entry:
; FASTPATH-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
; FASTPATH-NEXT:    ret i8 [[B]]
;
; ABORT-DYNAMIC-SHADOW-LABEL: define i8 @test_load_noattr
; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) {
; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
; ABORT-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
;
; RECOVER-DYNAMIC-SHADOW-LABEL: define i8 @test_load_noattr
; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) {
; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
;
; ABORT-ZERO-BASED-SHADOW-LABEL: define i8 @test_load_noattr
; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) {
; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i8 [[B]]
;
; RECOVER-ZERO-BASED-SHADOW-LABEL: define i8 @test_load_noattr
; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) {
; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i8 [[B]]
;
entry:
  %b = load i8, ptr %a, align 4
  ret i8 %b
}

define i8 @test_load_notmyattr(ptr %a) sanitize_address {
; CHECK-LABEL: define i8 @test_load_notmyattr
; CHECK-SAME: (ptr [[A:%.*]]) #[[ATTR1:[0-9]+]] {
; CHECK-NEXT:  entry:
; CHECK-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
; CHECK-NEXT:    ret i8 [[B]]
;
; NOFASTPATH-LABEL: define i8 @test_load_notmyattr
; NOFASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR1:[0-9]+]] {
; NOFASTPATH-NEXT:  entry:
; NOFASTPATH-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
; NOFASTPATH-NEXT:    ret i8 [[B]]
;
; FASTPATH-LABEL: define i8 @test_load_notmyattr
; FASTPATH-SAME: (ptr [[A:%.*]]) #[[ATTR1:[0-9]+]] {
; FASTPATH-NEXT:  entry:
; FASTPATH-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
; FASTPATH-NEXT:    ret i8 [[B]]
;
; ABORT-DYNAMIC-SHADOW-LABEL: define i8 @test_load_notmyattr
; ABORT-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR1:[0-9]+]] {
; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
; ABORT-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
;
; RECOVER-DYNAMIC-SHADOW-LABEL: define i8 @test_load_notmyattr
; RECOVER-DYNAMIC-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR1:[0-9]+]] {
; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
;
; ABORT-ZERO-BASED-SHADOW-LABEL: define i8 @test_load_notmyattr
; ABORT-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR1:[0-9]+]] {
; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i8 [[B]]
;
; RECOVER-ZERO-BASED-SHADOW-LABEL: define i8 @test_load_notmyattr
; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr [[A:%.*]]) #[[ATTR1:[0-9]+]] {
; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr [[A]], align 4
; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i8 [[B]]
;
entry:
  %b = load i8, ptr %a, align 4
  ret i8 %b
}

define i8 @test_load_addrspace(ptr addrspace(256) %a) sanitize_hwaddress {
; CHECK-LABEL: define i8 @test_load_addrspace
; CHECK-SAME: (ptr addrspace(256) [[A:%.*]]) #[[ATTR0]] {
; CHECK-NEXT:  entry:
; CHECK-NEXT:    [[B:%.*]] = load i8, ptr addrspace(256) [[A]], align 4
; CHECK-NEXT:    ret i8 [[B]]
;
; NOFASTPATH-LABEL: define i8 @test_load_addrspace
; NOFASTPATH-SAME: (ptr addrspace(256) [[A:%.*]]) #[[ATTR0]] {
; NOFASTPATH-NEXT:  entry:
; NOFASTPATH-NEXT:    [[B:%.*]] = load i8, ptr addrspace(256) [[A]], align 4
; NOFASTPATH-NEXT:    ret i8 [[B]]
;
; FASTPATH-LABEL: define i8 @test_load_addrspace
; FASTPATH-SAME: (ptr addrspace(256) [[A:%.*]]) #[[ATTR0]] {
; FASTPATH-NEXT:  entry:
; FASTPATH-NEXT:    [[B:%.*]] = load i8, ptr addrspace(256) [[A]], align 4
; FASTPATH-NEXT:    ret i8 [[B]]
;
; ABORT-DYNAMIC-SHADOW-LABEL: define i8 @test_load_addrspace
; ABORT-DYNAMIC-SHADOW-SAME: (ptr addrspace(256) [[A:%.*]]) #[[ATTR0]] {
; ABORT-DYNAMIC-SHADOW-NEXT:  entry:
; ABORT-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr addrspace(256) [[A]], align 4
; ABORT-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
;
; RECOVER-DYNAMIC-SHADOW-LABEL: define i8 @test_load_addrspace
; RECOVER-DYNAMIC-SHADOW-SAME: (ptr addrspace(256) [[A:%.*]]) #[[ATTR0]] {
; RECOVER-DYNAMIC-SHADOW-NEXT:  entry:
; RECOVER-DYNAMIC-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr addrspace(256) [[A]], align 4
; RECOVER-DYNAMIC-SHADOW-NEXT:    ret i8 [[B]]
;
; ABORT-ZERO-BASED-SHADOW-LABEL: define i8 @test_load_addrspace
; ABORT-ZERO-BASED-SHADOW-SAME: (ptr addrspace(256) [[A:%.*]]) #[[ATTR0]] {
; ABORT-ZERO-BASED-SHADOW-NEXT:  entry:
; ABORT-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr addrspace(256) [[A]], align 4
; ABORT-ZERO-BASED-SHADOW-NEXT:    ret i8 [[B]]
;
; RECOVER-ZERO-BASED-SHADOW-LABEL: define i8 @test_load_addrspace
; RECOVER-ZERO-BASED-SHADOW-SAME: (ptr addrspace(256) [[A:%.*]]) #[[ATTR0]] {
; RECOVER-ZERO-BASED-SHADOW-NEXT:  entry:
; RECOVER-ZERO-BASED-SHADOW-NEXT:    [[B:%.*]] = load i8, ptr addrspace(256) [[A]], align 4
; RECOVER-ZERO-BASED-SHADOW-NEXT:    ret i8 [[B]]
;
entry:
  %b = load i8, ptr addrspace(256) %a, align 4
  ret i8 %b
}