pgtrace_decoder.c   [plain text]

/*
 * Copyright (c) 2015 Apple Inc. All rights reserved.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_START@
 *
 * This file contains Original Code and/or Modifications of Original Code
 * as defined in and that are subject to the Apple Public Source License
 * Version 2.0 (the 'License'). You may not use this file except in
 * compliance with the License. The rights granted to you under the License
 * may not be used to create, or enable the creation or redistribution of,
 * unlawful or unlicensed copies of an Apple operating system, or to
 * circumvent, violate, or enable the circumvention or violation of, any
 * terms of an Apple operating system software license agreement.
 *
 * Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * The Original Code and all software distributed under the License are
 * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT.
 * Please see the License for the specific language governing rights and
 * limitations under the License.
 *
 * @APPLE_OSREFERENCE_LICENSE_HEADER_END@
 */

#if CONFIG_PGTRACE
#include <kern/debug.h>
#include <kern/clock.h>
#include <pexpert/pexpert.h>
#include <arm/pmap.h>
#include "pgtrace_decoder.h"

//-------------------------------------------------------------------
// Macros
//
#define DBG     1
#if DBG == 1
#define INLINE  __attribute__((noinline))
#else
#define INLINE  inline
#endif

#define BITS(v, msb, lsb)    ((v) << (31-msb) >> (31-msb) >> (lsb))
#define READ_GPR_X(ss, n, v) { \
    if (__builtin_expect(n < 31, 1)) (v) = (ss)->ss_64.x[(n)]; \
    else if (n == 31) (v) = 0; \
    else { panic("Invalid GPR x%d", n); __builtin_unreachable(); } \
}
#define READ_GPR_W(ss, n, v) { \
    if (__builtin_expect(n < 31, 1)) (v) = *(uint32_t*)&((ss)->ss_64.x[(n)]); \
    else if (n == 31) (v) = 0; \
    else { panic("Invalid GPR w%d", n); __builtin_unreachable(); } \
}
#define WRITE_GPR_X(ss, n, v) { \
    if (__builtin_expect(n < 31, 1)) (ss)->ss_64.x[(n)] = (v); \
    else if (n == 31) {} \
    else { panic("Invalid GPR x%d", n); __builtin_unreachable(); } \
}
#define WRITE_GPR_W(ss, n, v) { \
    if (__builtin_expect(n < 31, 1)) *(uint32_t*)&((ss)->ss_64.x[(n)]) = (v); \
    else if (n == 31) {} \
    else { panic("Invalid GPR w%d", n); __builtin_unreachable(); } \
}
#define SIGN_EXTEND_64(val, width)  (((int64_t)(val) << (64 - (width)) >> (64 - (width))))
#define ZERO_EXTEND_64(val, width)  (((uint64_t)(val) << (64 - (width))) >> (64 - (width)))

//-------------------------------------------------------------------
// Types
//
typedef int (*run_t)(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res);

typedef struct {
	vm_offset_t addr;
	uint64_t    bytes;
} instruction_info_t;

typedef bool (*get_info_t)(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info);

typedef struct {
	uint32_t mask;
	uint32_t value;
	run_t run;
	get_info_t get_info;
} type_entry_t;

//-------------------------------------------------------------------
// Statics
//
static int run_simd(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res);
static int run_c335(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res);
static int run_c336(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res);
static int run_c337(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res);
static int run_c338(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res);
static int run_c339(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res);
static int run_c3310(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res);
static int run_c3311(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res);
static int run_c3312(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res);
static int run_c3313(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res);
static int run_c3314(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res);
static int run_c3315(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res);
static int run_c3316(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res);
static bool get_info_simd(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info);
static bool get_info_c335(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info);
static bool get_info_c336(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info);
static bool get_info_c337(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info);
static bool get_info_c338(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info);
static bool get_info_c339(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info);
static bool get_info_c3310(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info);
static bool get_info_c3311(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info);
static bool get_info_c3312(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info);
static bool get_info_c3313(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info);
static bool get_info_c3314(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info);
static bool get_info_c3315(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info);
static bool get_info_c3316(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info);

// Table from ARM DDI 0487A.a C3.3
static type_entry_t typetbl[] = {
	{ 0x3f000000, 0x08000000, run_c336, get_info_c336 }, // Load/store exclusive
	{ 0x3b000000, 0x18000000, run_c335, get_info_c335 }, // Load register (literal)
	{ 0x3b800000, 0x28000000, run_c337, get_info_c337 }, // Load/store no-allocate pair (offset)
	{ 0x3b800000, 0x28800000, run_c3315, get_info_c3315 }, // Load/store register pair (post-indexed)
	{ 0x3b800000, 0x29000000, run_c3314, get_info_c3314 }, // Load/store register pair (offset)
	{ 0x3b800000, 0x29800000, run_c3316, get_info_c3316 }, // Load/store register pair (pre-indexed)
	{ 0x3b200c00, 0x38000000, run_c3312, get_info_c3312 }, // Load/store register (unscaled immediate)
	{ 0x3b200c00, 0x38000400, run_c338, get_info_c338 }, // Load/store register (immediate post-indexed)
	{ 0x3b200c00, 0x38000800, run_c3311, get_info_c3311 }, // Load/store register (unprivileged)
	{ 0x3b200c00, 0x38000c00, run_c339, get_info_c339 }, // Load/store register (immediate pre-indexed)
	{ 0x3b200c00, 0x38200800, run_c3310, get_info_c3310 }, // Load/store register (register offset)
	{ 0x3b000000, 0x39000000, run_c3313, get_info_c3313 }, // Load/store register (unsigned immediate)

	{ 0xbfbf0000, 0x0c000000, run_simd, get_info_simd }, // AdvSIMD load/store multiple structures
	{ 0xbfa00000, 0x0c800000, run_simd, get_info_simd }, // AdvSIMD load/store multiple structures (post-indexed)
	{ 0xbf980000, 0x0d000000, run_simd, get_info_simd }, // AdvSIMD load/store single structure
	{ 0xbf800000, 0x0d800000, run_simd, get_info_simd } // AdvSIMD load/store single structure (post-indexed)
};

static pgtrace_stats_t stats;

INLINE static void
do_str(uint8_t size, uint8_t Rt, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res)
{
	uint32_t wt;
	uint64_t xt;

	res->rr_rw = PGTRACE_RW_STORE;

	if (size == 8) {
		READ_GPR_X(ss, Rt, xt);
		res->rr_addrdata[0].ad_data = xt;
	} else {
		READ_GPR_W(ss, Rt, wt);
		res->rr_addrdata[0].ad_data = wt;
	}

	if (size == 1) {
		__asm__ volatile ("strb %w[wt], [%[va]]\n" :: [wt] "r"(wt), [va] "r"(va));
	} else if (size == 2) {
		__asm__ volatile ("strh %w[wt], [%[va]]\n" :: [wt] "r"(wt), [va] "r"(va));
	} else if (size == 4) {
		__asm__ volatile ("str %w[wt], [%[va]]\n" :: [wt] "r"(wt), [va] "r"(va));
	} else if (size == 8) {
		__asm__ volatile ("str %x[xt], [%[va]]\n" :: [xt] "r"(xt), [va] "r"(va));
	} else {
		panic("%s Invalid size %d\n", __func__, size);
	}

	stats.stat_decoder.sd_str++;
}

INLINE static void
do_ldr(uint8_t size, uint8_t Rt, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res)
{
	uint32_t wt;
	uint64_t xt;

	res->rr_rw = PGTRACE_RW_LOAD;

	if (size == 1) {
		__asm__ volatile ("ldrb %w[wt], [%[va]]\n" : [wt] "=r"(wt) : [va] "r"(va));
	} else if (size == 2) {
		__asm__ volatile ("ldrh %w[wt], [%[va]]\n" : [wt] "=r"(wt) : [va] "r"(va));
	} else if (size == 4) {
		__asm__ volatile ("ldr %w[wt], [%[va]]\n" : [wt] "=r"(wt) : [va] "r"(va));
	} else if (size == 8) {
		__asm__ volatile ("ldr %x[xt], [%[va]]\n" : [xt] "=r"(xt) : [va] "r"(va));
	} else {
		panic("%s Invalid size %d\n", __func__, size);
	}

	if (size == 8) {
		WRITE_GPR_X(ss, Rt, xt);
		res->rr_addrdata[0].ad_data = xt;
	} else {
		WRITE_GPR_W(ss, Rt, wt);
		res->rr_addrdata[0].ad_data = wt;
	}

	stats.stat_decoder.sd_ldr++;
}

INLINE static void
do_stp(uint8_t size, uint8_t Rt, uint8_t Rt2, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res)
{
	uint32_t wt1, wt2;
	uint64_t xt1, xt2;

	if (size == 4) {
		READ_GPR_W(ss, Rt, wt1);
		READ_GPR_W(ss, Rt2, wt2);
		__asm__ volatile ("stp %w[wt1], %w[wt2], [%[va]]\n" :: [wt1] "r"(wt1), [wt2] "r"(wt2), [va] "r"(va));
		res->rr_rw = PGTRACE_RW_STORE;
		res->rr_addrdata[1].ad_addr = va + sizeof(wt1);
		res->rr_addrdata[0].ad_data = wt1;
		res->rr_addrdata[1].ad_data = wt2;
	} else if (size == 8) {
		READ_GPR_X(ss, Rt, xt1);
		READ_GPR_X(ss, Rt2, xt2);
		__asm__ volatile ("stp %x[xt1], %x[xt2], [%[va]]\n" :: [xt1] "r"(xt1), [xt2] "r"(xt2), [va] "r"(va));
		res->rr_rw = PGTRACE_RW_STORE;
		res->rr_addrdata[1].ad_addr = va + sizeof(xt1);
		res->rr_addrdata[0].ad_data = xt1;
		res->rr_addrdata[1].ad_data = xt2;
	} else {
		panic("%s Invalid size %d\n", __func__, size);
	}

	stats.stat_decoder.sd_stp++;
}

INLINE static void
do_ldp(uint8_t size, uint8_t Rt, uint8_t Rt2, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res)
{
	uint32_t wt1, wt2;
	uint64_t xt1, xt2;

	if (size == 4) {
		__asm__ volatile ("ldp %w[wt1], %w[wt2], [%[va]]\n" : [wt1] "=r"(wt1), [wt2] "=r"(wt2) : [va] "r"(va));
		WRITE_GPR_W(ss, Rt, wt1);
		WRITE_GPR_W(ss, Rt2, wt2);
		res->rr_rw = PGTRACE_RW_STORE;
		res->rr_addrdata[1].ad_addr = va + sizeof(wt1);
		res->rr_addrdata[0].ad_data = wt1;
		res->rr_addrdata[1].ad_data = wt2;
	} else if (size == 8) {
		__asm__ volatile ("ldp %x[xt1], %x[xt2], [%[va]]\n" : [xt1] "=r"(xt1), [xt2] "=r"(xt2) : [va] "r"(va));
		WRITE_GPR_X(ss, Rt, xt1);
		WRITE_GPR_X(ss, Rt2, xt2);
		res->rr_rw = PGTRACE_RW_STORE;
		res->rr_addrdata[1].ad_addr = va + sizeof(xt1);
		res->rr_addrdata[0].ad_data = xt1;
		res->rr_addrdata[1].ad_data = xt2;
	} else {
		panic("%s Invalid size %d\n", __func__, size);
	}

	stats.stat_decoder.sd_ldp++;
}

INLINE static void
do_ldpsw(uint8_t Rt, uint8_t Rt2, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res)
{
	uint64_t xt1, xt2;

	__asm__ volatile ("ldpsw %x[xt1], %x[xt2], [%[va]]\n" : [xt1] "=r"(xt1), [xt2] "=r"(xt2) : [va] "r"(va));
	WRITE_GPR_X(ss, Rt, xt1);
	WRITE_GPR_X(ss, Rt2, xt2);
	res->rr_rw = PGTRACE_RW_LOAD;
	res->rr_addrdata[1].ad_addr = va + sizeof(uint32_t);
	res->rr_addrdata[0].ad_data = xt1;
	res->rr_addrdata[1].ad_data = xt2;

	stats.stat_decoder.sd_ldpsw++;
}

INLINE static void
do_ldrs(uint8_t size, uint8_t extsize, uint8_t Rt, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res)
{
	uint32_t wt;
	uint64_t xt;

	res->rr_rw = PGTRACE_RW_LOAD;

	if (size == 1 && extsize == 4) {
		__asm__ volatile ("ldrsb %w[wt], [%[va]]\n" : [wt] "=r"(wt) : [va] "r"(va));
	} else if (size == 1 && extsize == 8) {
		__asm__ volatile ("ldrsb %x[xt], [%[va]]\n" : [xt] "=r"(xt) : [va] "r"(va));
	} else if (size == 2 && extsize == 4) {
		__asm__ volatile ("ldrsh %w[wt], [%[va]]\n" : [wt] "=r"(wt) : [va] "r"(va));
	} else if (size == 2 && extsize == 8) {
		__asm__ volatile ("ldrsh %x[xt], [%[va]]\n" : [xt] "=r"(xt) : [va] "r"(va));
	} else if (size == 4 && extsize == 8) {
		__asm__ volatile ("ldrsw %x[xt], [%[va]]\n" : [xt] "=r"(xt) : [va] "r"(va));
	} else {
		panic("%s Invalid size %d extsize=%d\n", __func__, size, extsize);
	}

	if (extsize == 8) {
		WRITE_GPR_X(ss, Rt, xt);
		res->rr_addrdata[0].ad_data = xt;
	} else {
		WRITE_GPR_W(ss, Rt, wt);
		res->rr_addrdata[0].ad_data = wt;
	}

	stats.stat_decoder.sd_ldrs++;
}

INLINE static void
do_ldtrs(uint8_t size, uint8_t extsize, uint8_t Rt, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res)
{
	uint32_t wt;
	uint64_t xt;

	res->rr_rw = PGTRACE_RW_LOAD;

	if (size == 1 && extsize == 4) {
		__asm__ volatile ("ldtrsb %w[wt], [%[va]]\n" : [wt] "=r"(wt) : [va] "r"(va));
	} else if (size == 1 && extsize == 8) {
		__asm__ volatile ("ldtrsb %x[xt], [%[va]]\n" : [xt] "=r"(xt) : [va] "r"(va));
	} else if (size == 2 && extsize == 4) {
		__asm__ volatile ("ldtrsh %w[wt], [%[va]]\n" : [wt] "=r"(wt) : [va] "r"(va));
	} else if (size == 2 && extsize == 8) {
		__asm__ volatile ("ldtrsh %x[xt], [%[va]]\n" : [xt] "=r"(xt) : [va] "r"(va));
	} else if (size == 4 && extsize == 8) {
		__asm__ volatile ("ldtrsw %x[xt], [%[va]]\n" : [xt] "=r"(xt) : [va] "r"(va));
	} else {
		panic("%s Invalid size %d extsize=%d\n", __func__, size, extsize);
	}

	if (extsize == 8) {
		WRITE_GPR_X(ss, Rt, xt);
		res->rr_addrdata[0].ad_data = xt;
	} else {
		WRITE_GPR_W(ss, Rt, wt);
		res->rr_addrdata[0].ad_data = wt;
	}

	stats.stat_decoder.sd_ldtrs++;
}

INLINE static void
do_ldtr(uint8_t size, uint8_t Rt, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res)
{
	uint32_t wt;
	uint64_t xt;

	res->rr_rw = PGTRACE_RW_LOAD;

	if (size == 1) {
		__asm__ volatile ("ldtrb %w[wt], [%[va]]\n" : [wt] "=r"(wt) : [va] "r"(va));
	} else if (size == 2) {
		__asm__ volatile ("ldtrh %w[wt], [%[va]]\n" : [wt] "=r"(wt) : [va] "r"(va));
	} else if (size == 4) {
		__asm__ volatile ("ldtr %w[wt], [%[va]]\n" : [wt] "=r"(wt) : [va] "r"(va));
	} else if (size == 8) {
		__asm__ volatile ("ldtr %x[xt], [%[va]]\n" : [xt] "=r"(xt) : [va] "r"(va));
	} else {
		panic("%s Invalid size %d\n", __func__, size);
	}

	if (size == 8) {
		WRITE_GPR_X(ss, Rt, xt);
		res->rr_addrdata[0].ad_data = xt;
	} else {
		WRITE_GPR_W(ss, Rt, wt);
		res->rr_addrdata[0].ad_data = wt;
	}

	stats.stat_decoder.sd_ldtr++;
}

INLINE static void
do_sttr(uint8_t size, uint8_t Rt, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res)
{
	uint32_t wt;
	uint64_t xt;

	res->rr_rw = PGTRACE_RW_STORE;

	if (size == 8) {
		READ_GPR_X(ss, Rt, xt);
		res->rr_addrdata[0].ad_data = xt;
	} else {
		READ_GPR_W(ss, Rt, wt);
		res->rr_addrdata[0].ad_data = wt;
	}

	if (size == 1) {
		__asm__ volatile ("sttrb %w[wt], [%[va]]\n" :: [wt] "r"(wt), [va] "r"(va));
	} else if (size == 2) {
		__asm__ volatile ("sttrh %w[wt], [%[va]]\n" :: [wt] "r"(wt), [va] "r"(va));
	} else if (size == 4) {
		__asm__ volatile ("sttr %w[wt], [%[va]]\n" :: [wt] "r"(wt), [va] "r"(va));
	} else if (size == 8) {
		__asm__ volatile ("sttr %x[xt], [%[va]]\n" :: [xt] "r"(xt), [va] "r"(va));
	} else {
		panic("%s Invalid size %d\n", __func__, size);
	}

	stats.stat_decoder.sd_sttr++;
}

INLINE static void
do_prfm(uint8_t Rt, vm_offset_t va, pgtrace_run_result_t *res)
{
	if (Rt == 0) {
		__asm__ volatile ("prfm pldl1keep, [%[va]]\n" : : [va] "r"(va));
	} else if (Rt == 1) {
		__asm__ volatile ("prfm pldl1strm, [%[va]]\n" : : [va] "r"(va));
	} else if (Rt == 2) {
		__asm__ volatile ("prfm pldl2keep, [%[va]]\n" : : [va] "r"(va));
	} else if (Rt == 3) {
		__asm__ volatile ("prfm pldl2strm, [%[va]]\n" : : [va] "r"(va));
	} else if (Rt == 4) {
		__asm__ volatile ("prfm pldl3keep, [%[va]]\n" : : [va] "r"(va));
	} else if (Rt == 5) {
		__asm__ volatile ("prfm pldl3strm, [%[va]]\n" : : [va] "r"(va));
	} else if (Rt == 6) {
		__asm__ volatile ("prfm #6, [%[va]]\n" : : [va] "r"(va));
	} else if (Rt == 7) {
		__asm__ volatile ("prfm #7, [%[va]]\n" : : [va] "r"(va));
	} else if (Rt == 8) {
		__asm__ volatile ("prfm #8, [%[va]]\n" : : [va] "r"(va));
	} else if (Rt == 9) {
		__asm__ volatile ("prfm #9, [%[va]]\n" : : [va] "r"(va));
	} else if (Rt == 10) {
		__asm__ volatile ("prfm #10, [%[va]]\n" : : [va] "r"(va));
	} else if (Rt == 11) {
		__asm__ volatile ("prfm #11, [%[va]]\n" : : [va] "r"(va));
	} else if (Rt == 12) {
		__asm__ volatile ("prfm #12, [%[va]]\n" : : [va] "r"(va));
	} else if (Rt == 13) {
		__asm__ volatile ("prfm #13, [%[va]]\n" : : [va] "r"(va));
	} else if (Rt == 14) {
		__asm__ volatile ("prfm #14, [%[va]]\n" : : [va] "r"(va));
	} else if (Rt == 15) {
		__asm__ volatile ("prfm #15, [%[va]]\n" : : [va] "r"(va));
	} else if (Rt == 16) {
		__asm__ volatile ("prfm pstl1keep, [%[va]]\n" : : [va] "r"(va));
	} else if (Rt == 17) {
		__asm__ volatile ("prfm pstl1strm, [%[va]]\n" : : [va] "r"(va));
	} else if (Rt == 18) {
		__asm__ volatile ("prfm pstl2keep, [%[va]]\n" : : [va] "r"(va));
	} else if (Rt == 19) {
		__asm__ volatile ("prfm pstl2strm, [%[va]]\n" : : [va] "r"(va));
	} else if (Rt == 20) {
		__asm__ volatile ("prfm pstl3keep, [%[va]]\n" : : [va] "r"(va));
	} else if (Rt == 21) {
		__asm__ volatile ("prfm pstl3strm, [%[va]]\n" : : [va] "r"(va));
	} else if (Rt == 22) {
		__asm__ volatile ("prfm #22, [%[va]]\n" : : [va] "r"(va));
	} else if (Rt == 23) {
		__asm__ volatile ("prfm #23, [%[va]]\n" : : [va] "r"(va));
	} else if (Rt == 24) {
		__asm__ volatile ("prfm #24, [%[va]]\n" : : [va] "r"(va));
	} else if (Rt == 25) {
		__asm__ volatile ("prfm #25, [%[va]]\n" : : [va] "r"(va));
	} else if (Rt == 26) {
		__asm__ volatile ("prfm #26, [%[va]]\n" : : [va] "r"(va));
	} else if (Rt == 27) {
		__asm__ volatile ("prfm #27, [%[va]]\n" : : [va] "r"(va));
	} else if (Rt == 28) {
		__asm__ volatile ("prfm #28, [%[va]]\n" : : [va] "r"(va));
	} else if (Rt == 29) {
		__asm__ volatile ("prfm #29, [%[va]]\n" : : [va] "r"(va));
	} else if (Rt == 30) {
		__asm__ volatile ("prfm #30, [%[va]]\n" : : [va] "r"(va));
	} else if (Rt == 31) {
		__asm__ volatile ("prfm #31, [%[va]]\n" : : [va] "r"(va));
	} else {
		panic("%s Invalid Rt %d\n", __func__, Rt);
	}

	res->rr_num = 0;
	res->rr_rw = PGTRACE_RW_PREFETCH;

	stats.stat_decoder.sd_prfm++;
}

#define CANNOTDECODE(msg, inst) do {\
    panic("%s: " msg " inst=%x not supported yet\n", __func__, inst);\
} while (0)

static int
run_simd(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res)
{
#pragma unused(pa,va,ss,res)
	CANNOTDECODE("simd", inst);
	return 0;
}

static int
run_c335(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res)
{
	uint32_t opc = BITS(inst, 31, 30),
	    v = BITS(inst, 26, 26),
	    Rt = BITS(inst, 4, 0);
	uint8_t fields = (opc << 1) | v;

	res->rr_num = 1;
	res->rr_addrdata[0].ad_addr = pa;

	if (fields == 0) {
		do_ldr(4, Rt, va, ss, res);
	} else if ((fields == 1) ||
	    (fields == 3) ||
	    (fields == 5)) {
		CANNOTDECODE("simd", inst);
	} else if (fields == 2) {
		do_ldr(8, Rt, va, ss, res);
	} else if (fields == 4) {
		do_ldrs(4, 8, Rt, va, ss, res);
	} else if (fields == 6) {
		do_prfm(Rt, va, res);
	} else {
		CANNOTDECODE("unknown", inst);
	}

	stats.stat_decoder.sd_c335++;

	return 0;
}

static int
run_c336(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res)
{
	uint32_t ws, wt, wt1, wt2;
	uint64_t xt, xt1, xt2;
	uint32_t size = BITS(inst, 31, 30),
	    o2 = BITS(inst, 23, 23),
	    L = BITS(inst, 22, 22),
	    o1 = BITS(inst, 21, 21),
	    Rs = BITS(inst, 20, 16),
	    o0 = BITS(inst, 15, 15),
	    Rt2 = BITS(inst, 14, 10),
	    Rt = BITS(inst, 4, 0);
	uint8_t fields = (size << 4) | (o2 << 3) | (L << 2) | (o1 << 1) | o0;

	kprintf("%s Load/store exclusive on device memory???n", __func__);

	res->rr_num = 1;
	res->rr_addrdata[0].ad_addr = pa;

	switch (fields) {
	case 0:
		READ_GPR_W(ss, Rt, wt);
		__asm__ volatile ("stxrb %w[ws], %w[wt], [%[va]]\n" : [ws] "=r"(ws) : [wt] "r"(wt), [va] "r"(va));
		WRITE_GPR_W(ss, Rs, ws);
		res->rr_rw = PGTRACE_RW_STORE;
		res->rr_addrdata[0].ad_data = wt;
		break;
	case 1:
		READ_GPR_W(ss, Rt, wt);
		__asm__ volatile ("stlxrb %w[ws], %w[wt], [%[va]]\n" : [ws] "=r"(ws) : [wt] "r"(wt), [va] "r"(va));
		WRITE_GPR_W(ss, Rs, ws);
		res->rr_rw = PGTRACE_RW_STORE;
		res->rr_addrdata[0].ad_data = wt;
		break;
	case 4:
		__asm__ volatile ("ldxrb %w[wt], [%[va]]\n" : [wt] "=r"(wt) : [va] "r"(va));
		WRITE_GPR_W(ss, Rt, wt);
		res->rr_rw = PGTRACE_RW_LOAD;
		res->rr_addrdata[0].ad_data = wt;
		break;
	case 5:
		__asm__ volatile ("ldaxrb %w[wt], [%[va]]\n" : [wt] "=r"(wt) : [va] "r"(va));
		WRITE_GPR_W(ss, Rt, wt);
		res->rr_rw = PGTRACE_RW_LOAD;
		res->rr_addrdata[0].ad_data = wt;
		break;
	case 9:
		READ_GPR_W(ss, Rt, wt);
		__asm__ volatile ("stlrb %w[wt], [%[va]]\n" : [wt] "=r"(wt) : [va] "r"(va));
		res->rr_rw = PGTRACE_RW_STORE;
		res->rr_addrdata[0].ad_data = wt;
		break;
	case 0xd:
		__asm__ volatile ("ldarb %w[wt], [%[va]]\n" : [wt] "=r"(wt) : [va] "r"(va));
		WRITE_GPR_W(ss, Rt, wt);
		res->rr_rw = PGTRACE_RW_LOAD;
		res->rr_addrdata[0].ad_data = wt;
		break;
	case 0x10:
		READ_GPR_W(ss, Rt, wt);
		__asm__ volatile ("stxrh %w[ws], %w[wt], [%[va]]\n" : [ws] "=r"(ws) : [wt] "r"(wt), [va] "r"(va));
		WRITE_GPR_W(ss, Rs, ws);
		res->rr_rw = PGTRACE_RW_STORE;
		res->rr_addrdata[0].ad_data = wt;
		break;
	case 0x11:
		READ_GPR_W(ss, Rt, wt);
		__asm__ volatile ("stlxrh %w[ws], %w[wt], [%[va]]\n" : [ws] "=r"(ws) : [wt] "r"(wt), [va] "r"(va));
		WRITE_GPR_W(ss, Rs, ws);
		res->rr_rw = PGTRACE_RW_STORE;
		res->rr_addrdata[0].ad_data = wt;
		break;
	case 0x14:
		__asm__ volatile ("ldxrh %w[wt], [%[va]]\n" : [wt] "=r"(wt) : [va] "r"(va));
		WRITE_GPR_W(ss, Rt, wt);
		res->rr_rw = PGTRACE_RW_LOAD;
		res->rr_addrdata[0].ad_data = wt;
		break;
	case 0x15:
		__asm__ volatile ("ldaxrh %w[wt], [%[va]]\n" : [wt] "=r"(wt) : [va] "r"(va));
		WRITE_GPR_W(ss, Rt, wt);
		res->rr_rw = PGTRACE_RW_LOAD;
		res->rr_addrdata[0].ad_data = wt;
		break;
	case 0x19:
		READ_GPR_W(ss, Rt, wt);
		__asm__ volatile ("stlrh %w[wt], [%[va]]\n" : [wt] "=r"(wt) : [va] "r"(va));
		res->rr_rw = PGTRACE_RW_STORE;
		res->rr_addrdata[0].ad_data = wt;
		break;
	case 0x1d:
		__asm__ volatile ("ldarh %w[wt], [%[va]]\n" : [wt] "=r"(wt) : [va] "r"(va));
		WRITE_GPR_W(ss, Rt, wt);
		res->rr_rw = PGTRACE_RW_LOAD;
		res->rr_addrdata[0].ad_data = wt;
		break;
	case 0x20:
		READ_GPR_W(ss, Rt, wt);
		__asm__ volatile ("stxr %w[ws], %w[wt], [%[va]]\n" : [ws] "=r"(ws) : [wt] "r"(wt), [va] "r"(va));
		WRITE_GPR_W(ss, Rs, ws);
		res->rr_rw = PGTRACE_RW_STORE;
		res->rr_addrdata[0].ad_data = wt;
		break;
	case 0x21:
		READ_GPR_W(ss, Rt, wt);
		__asm__ volatile ("stlxr %w[ws], %w[wt], [%[va]]\n" : [ws] "=r"(ws) : [wt] "r"(wt), [va] "r"(va));
		WRITE_GPR_W(ss, Rs, ws);
		res->rr_rw = PGTRACE_RW_STORE;
		res->rr_addrdata[0].ad_data = wt;
		break;
	case 0x22:
		READ_GPR_W(ss, Rt, wt1);
		READ_GPR_W(ss, Rt2, wt2);
		__asm__ volatile ("stxp %w[ws], %w[wt1], %w[wt2], [%[va]]\n" : [ws] "=r"(ws) : [wt1] "r"(wt1), [wt2] "r"(wt2), [va] "r"(va));
		WRITE_GPR_W(ss, Rs, ws);
		res->rr_rw = PGTRACE_RW_STORE;
		res->rr_num = 2;
		res->rr_addrdata[0].ad_addr = va;
		res->rr_addrdata[1].ad_addr = va + sizeof(wt1);
		res->rr_addrdata[0].ad_data = wt1;
		res->rr_addrdata[1].ad_data = wt2;
		break;
	case 0x23:
		READ_GPR_W(ss, Rt, wt1);
		READ_GPR_W(ss, Rt2, wt2);
		__asm__ volatile ("stlxp %w[ws], %w[wt1], %w[wt2], [%[va]]\n" : [ws] "=r"(ws) : [wt1] "r"(wt1), [wt2] "r"(wt2), [va] "r"(va));
		WRITE_GPR_W(ss, Rs, ws);
		res->rr_rw = PGTRACE_RW_STORE;
		res->rr_num = 2;
		res->rr_addrdata[0].ad_addr = va;
		res->rr_addrdata[1].ad_addr = va + sizeof(wt1);
		res->rr_addrdata[0].ad_data = wt1;
		res->rr_addrdata[1].ad_data = wt2;
		break;
	case 0x24:
		__asm__ volatile ("ldxr %w[wt], [%[va]]\n" : [wt] "=r"(wt) : [va] "r"(va));
		WRITE_GPR_W(ss, Rt, wt);
		res->rr_rw = PGTRACE_RW_LOAD;
		res->rr_addrdata[0].ad_data = wt;
		break;
	case 0x25:
		__asm__ volatile ("ldaxr %w[wt], [%[va]]\n" : [wt] "=r"(wt) : [va] "r"(va));
		WRITE_GPR_W(ss, Rt, wt);
		res->rr_rw = PGTRACE_RW_LOAD;
		res->rr_addrdata[0].ad_data = wt;
		break;
	case 0x26:
		__asm__ volatile ("ldxp %w[wt1], %w[wt2], [%[va]]\n" : [wt1] "=r"(wt1), [wt2] "=r"(wt2) : [va] "r"(va));
		WRITE_GPR_W(ss, Rt, wt1);
		WRITE_GPR_W(ss, Rt2, wt2);
		res->rr_rw = PGTRACE_RW_LOAD;
		res->rr_num = 2;
		res->rr_addrdata[0].ad_addr = va;
		res->rr_addrdata[1].ad_addr = va + sizeof(wt1);
		res->rr_addrdata[0].ad_data = wt1;
		res->rr_addrdata[1].ad_data = wt2;
		break;
	case 0x27:
		__asm__ volatile ("ldaxp %w[wt1], %w[wt2], [%[va]]\n" : [wt1] "=r"(wt1), [wt2] "=r"(wt2) : [va] "r"(va));
		WRITE_GPR_W(ss, Rt, wt1);
		WRITE_GPR_W(ss, Rt2, wt2);
		res->rr_rw = PGTRACE_RW_LOAD;
		res->rr_num = 2;
		res->rr_addrdata[0].ad_addr = va;
		res->rr_addrdata[1].ad_addr = va + sizeof(wt1);
		res->rr_addrdata[0].ad_data = wt1;
		res->rr_addrdata[1].ad_data = wt2;
		break;
	case 0x29:
		READ_GPR_W(ss, Rt, wt);
		__asm__ volatile ("stlr %w[wt], [%[va]]\n" : [wt] "=r"(wt) : [va] "r"(va));
		res->rr_rw = PGTRACE_RW_STORE;
		res->rr_addrdata[0].ad_data = wt;
		break;
	case 0x2d:
		__asm__ volatile ("ldar %w[wt], [%[va]]\n" : [wt] "=r"(wt) : [va] "r"(va));
		WRITE_GPR_W(ss, Rt, wt);
		res->rr_rw = PGTRACE_RW_LOAD;
		res->rr_addrdata[0].ad_data = wt;
		break;
	case 0x30:
		READ_GPR_X(ss, Rt, xt);
		__asm__ volatile ("stxr %w[ws], %[xt], [%[va]]\n" : [ws] "=r"(ws) : [xt] "r"(xt), [va] "r"(va));
		WRITE_GPR_W(ss, Rs, ws);
		res->rr_rw = PGTRACE_RW_STORE;
		res->rr_addrdata[0].ad_data = xt;
		break;
	case 0x31:
		READ_GPR_X(ss, Rt, xt);
		__asm__ volatile ("stlxr %w[ws], %[xt], [%[va]]\n" : [ws] "=r"(ws) : [xt] "r"(xt), [va] "r"(va));
		WRITE_GPR_W(ss, Rs, ws);
		res->rr_rw = PGTRACE_RW_STORE;
		res->rr_addrdata[0].ad_data = xt;
		break;
	case 0x32:
		READ_GPR_X(ss, Rt, xt1);
		READ_GPR_X(ss, Rt2, xt2);
		__asm__ volatile ("stxp %w[ws], %[xt1], %[xt2], [%[va]]\n" : [ws] "=r"(ws) : [xt1] "r"(xt1), [xt2] "r"(xt2), [va] "r"(va));
		WRITE_GPR_W(ss, Rs, ws);
		res->rr_rw = PGTRACE_RW_STORE;
		res->rr_num = 2;
		res->rr_addrdata[0].ad_addr = va;
		res->rr_addrdata[1].ad_addr = va + sizeof(xt1);
		res->rr_addrdata[0].ad_data = xt1;
		res->rr_addrdata[1].ad_data = xt2;
		break;
	case 0x33:
		READ_GPR_X(ss, Rt, xt1);
		READ_GPR_X(ss, Rt2, xt2);
		__asm__ volatile ("stlxp %w[ws], %[xt1], %[xt2], [%[va]]\n" : [ws] "=r"(ws) : [xt1] "r"(xt1), [xt2] "r"(xt2), [va] "r"(va));
		WRITE_GPR_W(ss, Rs, ws);
		res->rr_rw = PGTRACE_RW_STORE;
		res->rr_num = 2;
		res->rr_addrdata[0].ad_addr = va;
		res->rr_addrdata[1].ad_addr = va + sizeof(xt1);
		res->rr_addrdata[0].ad_data = xt1;
		res->rr_addrdata[1].ad_data = xt2;
		break;
	case 0x34:
		__asm__ volatile ("ldxr %[xt], [%[va]]\n" : [xt] "=r"(xt) : [va] "r"(va));
		WRITE_GPR_X(ss, Rt, xt);
		res->rr_rw = PGTRACE_RW_LOAD;
		res->rr_addrdata[0].ad_data = xt;
		break;
	case 0x35:
		__asm__ volatile ("ldaxr %[xt], [%[va]]\n" : [xt] "=r"(xt) : [va] "r"(va));
		WRITE_GPR_X(ss, Rt, xt);
		res->rr_rw = PGTRACE_RW_LOAD;
		res->rr_addrdata[0].ad_data = xt;
		break;
	case 0x36:
		__asm__ volatile ("ldxp %[xt1], %[xt2], [%[va]]\n" : [xt1] "=r"(xt1), [xt2] "=r"(xt2) : [va] "r"(va));
		WRITE_GPR_X(ss, Rt, xt1);
		WRITE_GPR_X(ss, Rt2, xt2);
		res->rr_rw = PGTRACE_RW_LOAD;
		res->rr_num = 2;
		res->rr_addrdata[0].ad_addr = va;
		res->rr_addrdata[1].ad_addr = va + sizeof(xt1);
		res->rr_addrdata[0].ad_data = xt1;
		res->rr_addrdata[0].ad_data = xt2;
		break;
	case 0x37:
		__asm__ volatile ("ldaxp %[xt1], %[xt2], [%[va]]\n" : [xt1] "=r"(xt1), [xt2] "=r"(xt2) : [va] "r"(va));
		WRITE_GPR_X(ss, Rt, xt1);
		WRITE_GPR_X(ss, Rt2, xt2);
		res->rr_rw = PGTRACE_RW_LOAD;
		res->rr_num = 2;
		res->rr_addrdata[0].ad_addr = va;
		res->rr_addrdata[1].ad_addr = va + sizeof(xt1);
		res->rr_addrdata[0].ad_data = xt1;
		res->rr_addrdata[0].ad_data = xt2;
		break;
	case 0x39:
		READ_GPR_X(ss, Rt, xt);
		__asm__ volatile ("stlr %[xt], [%[va]]\n" : [xt] "=r"(xt) : [va] "r"(va));
		res->rr_rw = PGTRACE_RW_STORE;
		res->rr_addrdata[0].ad_data = xt;
		break;
	case 0x3d:
		__asm__ volatile ("ldar %[xt], [%[va]]\n" : [xt] "=r"(xt) : [va] "r"(va));
		WRITE_GPR_X(ss, Rt, xt);
		res->rr_rw = PGTRACE_RW_LOAD;
		res->rr_addrdata[0].ad_data = xt;
		break;
	default:
		CANNOTDECODE("unknown", inst);
	}

	stats.stat_decoder.sd_c336++;

	return 0;
}

static int
run_c337(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res)
{
	uint32_t wt1, wt2;
	uint64_t xt1, xt2;
	uint32_t opc = BITS(inst, 31, 30),
	    V = BITS(inst, 26, 26),
	    L = BITS(inst, 22, 22),
	    Rt = BITS(inst, 4, 0),
	    Rt2 = BITS(inst, 14, 10);
	uint8_t fields = (opc << 2) | (V << 1) | L;

	switch (fields) {
	case 0:
		READ_GPR_W(ss, Rt, wt1);
		READ_GPR_W(ss, Rt2, wt2);
		__asm__ volatile ("stnp %w[wt1], %w[wt2], [%[va]]\n" :: [wt1] "r"(wt1), [wt2] "r"(wt2), [va] "r"(va));
		res->rr_rw = PGTRACE_RW_STORE;
		res->rr_num = 2;
		res->rr_addrdata[0].ad_addr = pa;
		res->rr_addrdata[1].ad_addr = pa + sizeof(wt1);
		res->rr_addrdata[0].ad_data = wt1;
		res->rr_addrdata[1].ad_data = wt2;
		break;
	case 1:
		__asm__ volatile ("ldnp %w[wt1], %w[wt2], [%[va]]\n" : [wt1] "=r"(wt1), [wt2] "=r"(wt2) : [va] "r"(va));
		WRITE_GPR_W(ss, Rt, wt1);
		WRITE_GPR_W(ss, Rt2, wt2);
		res->rr_rw = PGTRACE_RW_STORE;
		res->rr_num = 2;
		res->rr_addrdata[0].ad_addr = pa;
		res->rr_addrdata[1].ad_addr = pa + sizeof(wt1);
		res->rr_addrdata[0].ad_data = wt1;
		res->rr_addrdata[1].ad_data = wt2;
		break;
	case 2:
	case 3:
	case 6:
	case 7:
	case 10:
	case 11:
		CANNOTDECODE("simd", inst);
	case 8:
		READ_GPR_X(ss, Rt, xt1);
		READ_GPR_X(ss, Rt2, xt2);
		__asm__ volatile ("stnp %x[xt1], %x[xt2], [%[va]]\n" :: [xt1] "r"(xt1), [xt2] "r"(xt2), [va] "r"(va));
		res->rr_rw = PGTRACE_RW_STORE;
		res->rr_num = 2;
		res->rr_addrdata[0].ad_addr = pa;
		res->rr_addrdata[1].ad_addr = pa + sizeof(xt1);
		res->rr_addrdata[0].ad_data = xt1;
		res->rr_addrdata[1].ad_data = xt2;
		break;
	case 9:
		__asm__ volatile ("ldnp %x[xt1], %x[xt2], [%[va]]\n" : [xt1] "=r"(xt1), [xt2] "=r"(xt2) : [va] "r"(va));
		WRITE_GPR_X(ss, Rt, xt1);
		WRITE_GPR_X(ss, Rt2, xt2);
		res->rr_rw = PGTRACE_RW_STORE;
		res->rr_num = 2;
		res->rr_addrdata[0].ad_addr = pa;
		res->rr_addrdata[1].ad_addr = pa + sizeof(xt1);
		res->rr_addrdata[0].ad_data = xt1;
		res->rr_addrdata[1].ad_data = xt2;
		break;
	default:
		CANNOTDECODE("simd", inst);
	}

	stats.stat_decoder.sd_c337++;

	return 0;
}

static int
run_c338(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res)
{
	uint32_t size = BITS(inst, 31, 30),
	    V = BITS(inst, 26, 26),
	    opc = BITS(inst, 23, 22),
	    Rt = BITS(inst, 4, 0);
	uint8_t fields = (size << 3) | (V << 2) | opc;

	res->rr_num = 1;
	res->rr_addrdata[0].ad_addr = pa;

	if (fields == 0) {
		do_str(1, Rt, va, ss, res);
	} else if (fields == 1) {
		do_ldr(1, Rt, va, ss, res);
	} else if (fields == 2) {
		do_ldrs(1, 8, Rt, va, ss, res);
	} else if (fields == 3) {
		do_ldrs(1, 4, Rt, va, ss, res);
	} else if ((fields == 4) ||
	    (fields == 5) ||
	    (fields == 6) ||
	    (fields == 7) ||
	    (fields == 12) ||
	    (fields == 13) ||
	    (fields == 0x14) ||
	    (fields == 0x15) ||
	    (fields == 0x1c) ||
	    (fields == 0x1d)) {
		CANNOTDECODE("simd", inst);
	} else if (fields == 8) {
		do_str(2, Rt, va, ss, res);
	} else if (fields == 9) {
		do_ldr(2, Rt, va, ss, res);
	} else if (fields == 10) {
		do_ldrs(2, 8, Rt, va, ss, res);
	} else if (fields == 11) {
		do_ldrs(2, 4, Rt, va, ss, res);
	} else if (fields == 0x10) {
		do_str(4, Rt, va, ss, res);
	} else if (fields == 0x11) {
		do_ldr(4, Rt, va, ss, res);
	} else if (fields == 0x12) {
		do_ldrs(4, 8, Rt, va, ss, res);
	} else if (fields == 0x18) {
		do_str(8, Rt, va, ss, res);
	} else if (fields == 0x19) {
		do_ldr(8, Rt, va, ss, res);
	} else {
		CANNOTDECODE("unknown", inst);
	}

	stats.stat_decoder.sd_c338++;

	return 0;
}

static int
run_c339(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res)
{
	uint32_t size = BITS(inst, 31, 30),
	    V = BITS(inst, 26, 26),
	    opc = BITS(inst, 23, 22),
	    Rt = BITS(inst, 4, 0);
	uint8_t fields = (size << 3) | (V << 2) | opc;

	res->rr_num = 1;
	res->rr_addrdata[0].ad_addr = pa;

	if (fields == 0) {
		do_str(1, Rt, va, ss, res);
	} else if (fields == 1) {
		do_ldr(1, Rt, va, ss, res);
	} else if (fields == 2) {
		do_ldrs(1, 8, Rt, va, ss, res);
	} else if (fields == 3) {
		do_ldrs(1, 4, Rt, va, ss, res);
	} else if ((fields == 4) ||
	    (fields == 5) ||
	    (fields == 6) ||
	    (fields == 7) ||
	    (fields == 12) ||
	    (fields == 13) ||
	    (fields == 0x14) ||
	    (fields == 0x15) ||
	    (fields == 0x1c) ||
	    (fields == 0x1d)) {
		CANNOTDECODE("simd", inst);
	} else if (fields == 8) {
		do_str(2, Rt, va, ss, res);
	} else if (fields == 9) {
		do_ldr(2, Rt, va, ss, res);
	} else if (fields == 10) {
		do_ldrs(2, 8, Rt, va, ss, res);
	} else if (fields == 11) {
		do_ldrs(2, 4, Rt, va, ss, res);
	} else if (fields == 0x10) {
		do_str(4, Rt, va, ss, res);
	} else if (fields == 0x11) {
		do_ldr(4, Rt, va, ss, res);
	} else if (fields == 0x12) {
		do_ldrs(4, 8, Rt, va, ss, res);
	} else if (fields == 0x18) {
		do_str(8, Rt, va, ss, res);
	} else if (fields == 0x19) {
		do_ldr(8, Rt, va, ss, res);
	} else {
		CANNOTDECODE("unknown", inst);
	}

	stats.stat_decoder.sd_c339++;

	return 0;
}

static int
run_c3310(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res)
{
	uint32_t size = BITS(inst, 31, 30),
	    V = BITS(inst, 26, 26),
	    opc = BITS(inst, 23, 22),
	    Rt = BITS(inst, 4, 0);
	uint8_t fields = (size << 3) | (V << 2) | opc;

	res->rr_num = 1;
	res->rr_addrdata[0].ad_addr = pa;

	if (fields == 0) {
		do_str(1, Rt, va, ss, res);
	} else if (fields == 1) {
		do_ldr(1, Rt, va, ss, res);
	} else if (fields == 2) {
		do_ldrs(1, 8, Rt, va, ss, res);
	} else if (fields == 3) {
		do_ldrs(1, 4, Rt, va, ss, res);
	} else if ((fields == 4) ||
	    (fields == 5) ||
	    (fields == 6) ||
	    (fields == 7) ||
	    (fields == 12) ||
	    (fields == 13) ||
	    (fields == 0x14) ||
	    (fields == 0x15) ||
	    (fields == 0x1c) ||
	    (fields == 0x1d)) {
		CANNOTDECODE("simd", inst);
	} else if (fields == 8) {
		do_str(2, Rt, va, ss, res);
	} else if (fields == 9) {
		do_ldr(2, Rt, va, ss, res);
	} else if (fields == 10) {
		do_ldrs(2, 8, Rt, va, ss, res);
	} else if (fields == 11) {
		do_ldrs(2, 4, Rt, va, ss, res);
	} else if (fields == 0x10) {
		do_str(4, Rt, va, ss, res);
	} else if (fields == 0x11) {
		do_ldr(4, Rt, va, ss, res);
	} else if (fields == 0x12) {
		do_ldrs(4, 8, Rt, va, ss, res);
	} else if (fields == 0x18) {
		do_str(8, Rt, va, ss, res);
	} else if (fields == 0x19) {
		do_ldr(8, Rt, va, ss, res);
	} else if (fields == 0x1a) {
		do_prfm(Rt, va, res);
	} else {
		CANNOTDECODE("unknown", inst);
	}

	stats.stat_decoder.sd_c3310++;

	return 0;
}

static int
run_c3311(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res)
{
	uint32_t size = BITS(inst, 31, 30),
	    V = BITS(inst, 26, 26),
	    opc = BITS(inst, 23, 22),
	    Rt = BITS(inst, 4, 0);
	uint8_t fields = (size << 3) | (V << 2) | opc;

	res->rr_num = 1;
	res->rr_addrdata[0].ad_addr = pa;

	if (fields == 0) {
		do_sttr(1, Rt, va, ss, res);
	} else if (fields == 1) {
		do_ldtr(1, Rt, va, ss, res);
	} else if (fields == 2) {
		do_ldtrs(1, 8, Rt, va, ss, res);
	} else if (fields == 3) {
		do_ldtrs(1, 4, Rt, va, ss, res);
	} else if (fields == 8) {
		do_sttr(2, Rt, va, ss, res);
	} else if (fields == 9) {
		do_ldtr(2, Rt, va, ss, res);
	} else if (fields == 10) {
		do_ldtrs(2, 8, Rt, va, ss, res);
	} else if (fields == 11) {
		do_ldtrs(2, 4, Rt, va, ss, res);
	} else if (fields == 0x10) {
		do_sttr(4, Rt, va, ss, res);
	} else if (fields == 0x11) {
		do_ldtr(4, Rt, va, ss, res);
	} else if (fields == 0x12) {
		do_ldtrs(4, 8, Rt, va, ss, res);
	} else if (fields == 0x18) {
		do_sttr(8, Rt, va, ss, res);
	} else if (fields == 0x19) {
		do_ldtr(8, Rt, va, ss, res);
	} else {
		CANNOTDECODE("unknown", inst);
	}

	stats.stat_decoder.sd_c3311++;

	return 0;
}

static int
run_c3312(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res)
{
	uint32_t size = BITS(inst, 31, 30),
	    V = BITS(inst, 26, 26),
	    opc = BITS(inst, 23, 22),
	    Rt = BITS(inst, 4, 0);
	uint8_t fields = (size << 3) | (V << 2) | opc;

	res->rr_num = 1;
	res->rr_addrdata[0].ad_addr = pa;

	if (fields == 0) {
		do_str(1, Rt, va, ss, res);
	} else if (fields == 1) {
		do_ldr(1, Rt, va, ss, res);
	} else if (fields == 2) {
		do_ldrs(1, 8, Rt, va, ss, res);
	} else if (fields == 3) {
		do_ldrs(1, 4, Rt, va, ss, res);
	} else if ((fields == 4) ||
	    (fields == 5) ||
	    (fields == 6) ||
	    (fields == 7) ||
	    (fields == 12) ||
	    (fields == 13) ||
	    (fields == 0x14) ||
	    (fields == 0x15) ||
	    (fields == 0x1c) ||
	    (fields == 0x1d)) {
		CANNOTDECODE("simd", inst);
	} else if (fields == 8) {
		do_str(2, Rt, va, ss, res);
	} else if (fields == 9) {
		do_ldr(2, Rt, va, ss, res);
	} else if (fields == 10) {
		do_ldrs(2, 8, Rt, va, ss, res);
	} else if (fields == 11) {
		do_ldrs(2, 4, Rt, va, ss, res);
	} else if (fields == 0x10) {
		do_str(4, Rt, va, ss, res);
	} else if (fields == 0x11) {
		do_ldr(4, Rt, va, ss, res);
	} else if (fields == 0x12) {
		do_ldrs(4, 8, Rt, va, ss, res);
	} else if (fields == 0x18) {
		do_str(8, Rt, va, ss, res);
	} else if (fields == 0x19) {
		do_ldr(8, Rt, va, ss, res);
	} else if (fields == 0x1a) {
		do_prfm(Rt, va, res);
	} else {
		CANNOTDECODE("unknown", inst);
	}

	stats.stat_decoder.sd_c3312++;

	return 0;
}

static int
run_c3313(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res)
{
	uint32_t size = BITS(inst, 31, 30),
	    V = BITS(inst, 26, 26),
	    opc = BITS(inst, 23, 22),
	    Rt = BITS(inst, 4, 0);
	uint8_t fields = (size << 3) | (V << 2) | opc;

	res->rr_num = 1;
	res->rr_addrdata[0].ad_addr = pa;

	if (fields == 0) {
		do_str(1, Rt, va, ss, res);
	} else if (fields == 1) {
		do_ldr(1, Rt, va, ss, res);
	} else if (fields == 2) {
		do_ldrs(1, 8, Rt, va, ss, res);
	} else if (fields == 3) {
		do_ldrs(1, 4, Rt, va, ss, res);
	} else if ((fields == 4) ||
	    (fields == 5) ||
	    (fields == 6) ||
	    (fields == 7) ||
	    (fields == 12) ||
	    (fields == 13) ||
	    (fields == 0x14) ||
	    (fields == 0x15) ||
	    (fields == 0x1c) ||
	    (fields == 0x1d)) {
		CANNOTDECODE("simd", inst);
	} else if (fields == 8) {
		do_str(2, Rt, va, ss, res);
	} else if (fields == 9) {
		do_ldr(2, Rt, va, ss, res);
	} else if (fields == 10) {
		do_ldrs(2, 8, Rt, va, ss, res);
	} else if (fields == 11) {
		do_ldrs(2, 4, Rt, va, ss, res);
	} else if (fields == 0x10) {
		do_str(4, Rt, va, ss, res);
	} else if (fields == 0x11) {
		do_ldr(4, Rt, va, ss, res);
	} else if (fields == 0x12) {
		do_ldrs(4, 8, Rt, va, ss, res);
	} else if (fields == 0x18) {
		do_str(8, Rt, va, ss, res);
	} else if (fields == 0x19) {
		do_ldr(8, Rt, va, ss, res);
	} else if (fields == 0x1a) {
		do_prfm(Rt, va, res);
	} else {
		CANNOTDECODE("unknown", inst);
	}

	stats.stat_decoder.sd_c3313++;

	return 0;
}

static int
run_c3314(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res)
{
	uint32_t opc = BITS(inst, 31, 30),
	    V = BITS(inst, 26, 26),
	    L = BITS(inst, 22, 22),
	    Rt = BITS(inst, 4, 0),
	    Rt2 = BITS(inst, 14, 10);
	uint8_t fields = (opc << 2) | (V << 1) | L;

	res->rr_num = 2;
	res->rr_addrdata[0].ad_addr = pa;

	if (fields == 0) {
		do_stp(4, Rt, Rt2, va, ss, res);
	} else if (fields == 1) {
		do_ldp(4, Rt, Rt2, va, ss, res);
	} else if ((fields == 2) ||
	    (fields == 3) ||
	    (fields == 6) ||
	    (fields == 7) ||
	    (fields == 10) ||
	    (fields == 11)) {
		CANNOTDECODE("simd", inst);
	} else if (fields == 5) {
		do_ldpsw(Rt, Rt2, va, ss, res);
	} else if (fields == 8) {
		do_stp(8, Rt, Rt2, va, ss, res);
	} else if (fields == 9) {
		do_ldp(8, Rt, Rt2, va, ss, res);
	} else {
		CANNOTDECODE("unknown", inst);
	}

	stats.stat_decoder.sd_c3314++;

	return 0;
}

static int
run_c3315(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res)
{
	uint32_t opc = BITS(inst, 31, 30),
	    V = BITS(inst, 26, 26),
	    L = BITS(inst, 22, 22),
	    Rt = BITS(inst, 4, 0),
	    Rt2 = BITS(inst, 14, 10);
	uint8_t fields = (opc << 2) | (V << 1) | L;

	res->rr_num = 2;
	res->rr_addrdata[0].ad_addr = pa;

	if (fields == 0) {
		do_stp(4, Rt, Rt2, va, ss, res);
	} else if (fields == 1) {
		do_ldp(4, Rt, Rt2, va, ss, res);
	} else if ((fields == 2) ||
	    (fields == 3) ||
	    (fields == 6) ||
	    (fields == 7) ||
	    (fields == 10) ||
	    (fields == 11)) {
		CANNOTDECODE("simd", inst);
	} else if (fields == 5) {
		do_ldpsw(Rt, Rt2, va, ss, res);
	} else if (fields == 8) {
		do_stp(8, Rt, Rt2, va, ss, res);
	} else if (fields == 9) {
		do_ldp(8, Rt, Rt2, va, ss, res);
	} else {
		CANNOTDECODE("unknown", inst);
	}

	stats.stat_decoder.sd_c3315++;

	return 0;
}

static int
run_c3316(uint32_t inst, vm_offset_t pa, vm_offset_t va, arm_saved_state_t *ss, pgtrace_run_result_t *res)
{
	uint32_t opc = BITS(inst, 31, 30),
	    V = BITS(inst, 26, 26),
	    L = BITS(inst, 22, 22),
	    Rt = BITS(inst, 4, 0),
	    Rt2 = BITS(inst, 14, 10);
	uint8_t fields = (opc << 2) | (V << 1) | L;

	res->rr_num = 2;
	res->rr_addrdata[0].ad_addr = pa;

	if (fields == 0) {
		do_stp(4, Rt, Rt2, va, ss, res);
	} else if (fields == 1) {
		do_ldp(4, Rt, Rt2, va, ss, res);
	} else if ((fields == 2) ||
	    (fields == 3) ||
	    (fields == 6) ||
	    (fields == 7) ||
	    (fields == 10) ||
	    (fields == 11)) {
		CANNOTDECODE("simd", inst);
	} else if (fields == 5) {
		do_ldpsw(Rt, Rt2, va, ss, res);
	} else if (fields == 8) {
		do_stp(8, Rt, Rt2, va, ss, res);
	} else if (fields == 9) {
		do_ldp(8, Rt, Rt2, va, ss, res);
	} else {
		CANNOTDECODE("unknown", inst);
	}

	stats.stat_decoder.sd_c3316++;

	return 0;
}

static bool
get_info_simd(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info)
{
#pragma unused(inst, ss, info)
	CANNOTDECODE("simd", inst);
	return false;
}

// load register (literal)
static bool
get_info_c335(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info)
{
	uint32_t opc = BITS(inst, 31, 30);
	uint32_t V = BITS(inst, 26, 26);
	uint32_t imm19 = BITS(inst, 23, 5);
	uint32_t fields = (opc << 1) | V;
	uint8_t scale;

	if (__builtin_expect(fields > 6, false)) {
		CANNOTDECODE("invalid", inst);
		return false;
	}

	assert(fields <= 6);

	if (V == 1) {
		scale = 2 + opc;
	} else {
		switch (opc) {
		case 0 ... 1:
			scale = 2 + opc;
			break;
		case 2:
			scale = 2;
			break;
		default:
			CANNOTDECODE("invalid", inst);
			return false;
		}
	}

	info->bytes = 1 << scale;
	info->addr = ss->ss_64.pc + (SIGN_EXTEND_64(imm19, 19) << 2);

	return true;
}

// load/store exclusive
static bool
get_info_c336(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info)
{
	uint32_t size = BITS(inst, 31, 30);
	uint32_t o2 = BITS(inst, 23, 23);
	uint32_t L = BITS(inst, 22, 22);
	uint32_t o1 = BITS(inst, 21, 21);
	uint32_t o0 = BITS(inst, 15, 15);
	uint32_t Rn = BITS(inst, 9, 5);
	uint32_t fields = (size << 4) | (o2 << 3) | (L << 2) | (o1 << 1) | o0;

	if (__builtin_expect((2 <= fields && fields <= 3) ||
	    (6 <= fields && fields <= 8) ||
	    (10 <= fields && fields <= 12) ||
	    (14 <= fields && fields <= 15) ||
	    (18 <= fields && fields <= 19) ||
	    (22 <= fields && fields <= 24) ||
	    (26 <= fields && fields <= 28) ||
	    (30 <= fields && fields <= 31) ||
	    (40 == fields) ||
	    (42 <= fields && fields <= 44) ||
	    (46 <= fields && fields <= 47) ||
	    (56 == fields) ||
	    (58 <= fields && fields <= 60) ||
	    (62 <= fields), false)) {
		CANNOTDECODE("invalid", inst);
		return false;
	}

	info->bytes = (1 << size) << o1;
	info->addr = ss->ss_64.x[Rn];

	return true;
}

// load/store no-allocate pair (offset)
bool
get_info_c337(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info)
{
	uint32_t opc = BITS(inst, 31, 30);
	uint32_t V = BITS(inst, 26, 26);
	uint32_t L = BITS(inst, 22, 22);
	uint32_t imm7 = BITS(inst, 21, 15);
	uint32_t Rn = BITS(inst, 9, 5);
	uint32_t fields = (opc << 2) | (V << 1) | L;
	uint8_t scale;

	if (__builtin_expect((4 <= fields && fields <= 5) ||
	    (12 <= fields), false)) {
		CANNOTDECODE("invalid", inst);
		return false;
	}

	if (V == 1) {
		scale = opc + 2;
	} else {
		scale = BITS(opc, 1, 1) + 2;
	}

	// double since it's pair
	info->bytes = 2 * (1 << scale);
	info->addr = ss->ss_64.x[Rn] + (SIGN_EXTEND_64(imm7, 7) << scale);

	return true;
}

// load/store reigster (immediate post-indexed)
static bool
get_info_c338(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info)
{
	uint32_t size = BITS(inst, 31, 30);
	uint32_t V = BITS(inst, 26, 26);
	uint32_t opc = BITS(inst, 23, 22);
	uint32_t Rn = BITS(inst, 9, 5);
	uint32_t fields = (size << 3) | (V << 2) | opc;
	uint8_t scale;

	if (__builtin_expect((14 <= fields && fields <= 15) ||
	    (19 == fields) ||
	    (22 <= fields && fields <= 23) ||
	    (26 <= fields && fields <= 27) ||
	    (30 <= fields), false)) {
		CANNOTDECODE("invalid", inst);
		return false;
	}

	if (V == 1) {
		scale = BITS(opc, 1, 1) << 2 | size;
	} else {
		scale = size;
	}

	info->bytes = 1 << scale;
	// post-indexed
	info->addr = ss->ss_64.x[Rn];

	return true;
}

// load/store register (immediate pre-indexed)
static bool
get_info_c339(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info)
{
	uint32_t size = BITS(inst, 31, 30);
	uint32_t V = BITS(inst, 26, 26);
	uint32_t opc = BITS(inst, 23, 22);
	uint32_t imm9 = BITS(inst, 20, 12);
	uint32_t Rn = BITS(inst, 9, 5);
	uint32_t fields = (size << 3) | (V << 2) | opc;
	uint8_t scale;

	if (__builtin_expect((14 <= fields && fields <= 15) ||
	    (19 == fields) ||
	    (22 <= fields && fields <= 23) ||
	    (26 <= fields && fields <= 27) ||
	    (30 <= fields), false)) {
		CANNOTDECODE("invalid", inst);
		return false;
	}

	if (V == 1) {
		scale = BITS(opc, 1, 1) << 2 | size;
	} else {
		scale = size;
	}

	info->bytes = 1 << scale;
	info->addr = ss->ss_64.x[Rn] + SIGN_EXTEND_64(imm9, 9);

	return true;
}

// load/store register (register offset)
static bool
get_info_c3310(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info)
{
	uint32_t size = BITS(inst, 31, 30);
	uint32_t V = BITS(inst, 26, 26);
	uint32_t opc = BITS(inst, 23, 22);
	uint32_t Rm = BITS(inst, 20, 16);
	uint32_t option = BITS(inst, 15, 13);
	uint32_t S = BITS(inst, 12, 12);
	uint32_t Rn = BITS(inst, 9, 5);
	uint32_t fields = (size << 3) | (V << 2) | opc;
	uint32_t scale;

	if (__builtin_expect((14 <= fields && fields <= 15) ||
	    (19 == fields) ||
	    (22 <= fields && fields <= 23) ||
	    (27 == fields) ||
	    (30 <= fields), false)) {
		CANNOTDECODE("invalid", inst);
		return false;
	}

	if (V == 1) {
		scale = BITS(opc, 1, 1) | size;
	} else {
		scale = size;
	}

	info->bytes = 1 << scale;

	uint64_t m = ss->ss_64.x[Rm];
	uint8_t shift = (S == 1 ? scale : 0);

	switch (option) {
	case 0 ... 3:
		info->addr = ss->ss_64.x[Rn] + (ZERO_EXTEND_64(m, 8 << option) << shift);
		break;
	case 4 ... 7:
		info->addr = ss->ss_64.x[Rn] + (SIGN_EXTEND_64(m, 8 << BITS(option, 1, 0)) << shift);
		break;
	default:
		CANNOTDECODE("invalid", inst);
		return false;
	}

	return true;
}

// load/store register (unprivileged)
static bool
get_info_c3311(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info)
{
	uint32_t size = BITS(inst, 31, 30);
	uint32_t V = BITS(inst, 26, 26);
	uint32_t opc = BITS(inst, 23, 22);
	uint32_t imm9 = BITS(inst, 20, 12);
	uint32_t Rn = BITS(inst, 9, 5);
	uint32_t fields = (size << 3) | (V << 2) | opc;

	if (__builtin_expect((4 <= fields && fields <= 7) ||
	    (12 <= fields && fields <= 15) ||
	    (19 <= fields && fields <= 23) ||
	    (26 <= fields), false)) {
		CANNOTDECODE("invalid", inst);
		return false;
	}

	info->bytes = 1 << size;
	info->addr = ss->ss_64.x[Rn] + SIGN_EXTEND_64(imm9, 9);

	return true;
}

// load/store register (unscaled immediate)
static bool
get_info_c3312(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info)
{
	uint32_t size = BITS(inst, 31, 30);
	uint32_t V = BITS(inst, 26, 26);
	uint32_t opc = BITS(inst, 23, 22);
	uint32_t imm9 = BITS(inst, 20, 12);
	uint32_t Rn = BITS(inst, 9, 5);
	uint32_t fields = (size << 3) | (V << 2) | opc;
	uint32_t scale;

	if (__builtin_expect((14 <= fields && fields <= 15) ||
	    (19 == fields) ||
	    (22 <= fields && fields <= 23) ||
	    (27 == fields) ||
	    (30 <= fields), false)) {
		CANNOTDECODE("invalid", inst);
		return false;
	}

	if (V == 1) {
		scale = BITS(opc, 1, 1) << 2 | size;
	} else {
		scale = size;
	}

	info->bytes = 1 < scale;
	info->addr = ss->ss_64.x[Rn] + SIGN_EXTEND_64(imm9, 9);

	return true;
}

// load/store register (unsigned immediate)
bool
get_info_c3313(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info)
{
	uint32_t size = BITS(inst, 31, 30);
	uint32_t V = BITS(inst, 26, 26);
	uint32_t opc = BITS(inst, 23, 22);
	uint32_t imm12 = BITS(inst, 21, 10);
	uint32_t Rn = BITS(inst, 9, 5);
	uint32_t fields = (size << 3) | (V << 2) | opc;
	uint32_t scale;

	if (__builtin_expect((14 <= fields && fields <= 15) ||
	    (19 == fields) ||
	    (22 <= fields && fields <= 23) ||
	    (27 == fields) ||
	    (30 <= fields), false)) {
		CANNOTDECODE("invalid", inst);
		return false;
	}

	if (V == 1) {
		scale = BITS(opc, 1, 1) << 2 | size;
	} else {
		scale = size;
	}

	info->bytes = 1 << scale;
	info->addr = ss->ss_64.x[Rn] + (ZERO_EXTEND_64(imm12, 12) << scale);

	return true;
}

// load/store register pair (offset)
static bool
get_info_c3314(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info)
{
	uint32_t opc = BITS(inst, 31, 30);
	uint32_t V = BITS(inst, 26, 26);
	uint32_t L = BITS(inst, 22, 22);
	uint32_t imm7 = BITS(inst, 21, 15);
	uint32_t Rn = BITS(inst, 9, 5);
	uint32_t fields = (opc << 2) | (V << 1) | L;
	uint8_t scale = 2 + (opc >> 1);

	if (__builtin_expect((4 == fields) ||
	    (12 <= fields), false)) {
		CANNOTDECODE("invalid", inst);
		return false;
	}

	if (V == 1) {
		scale = 2 + opc;
	} else {
		scale = 2 + BITS(opc, 1, 1);
	}

	info->bytes = 2 * (1 << scale);
	info->addr = ss->ss_64.x[Rn] + (SIGN_EXTEND_64(imm7, 7) << scale);

	return true;
}

// load/store register pair (post-indexed)
static bool
get_info_c3315(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info)
{
	uint32_t opc = BITS(inst, 31, 30);
	uint32_t V = BITS(inst, 26, 26);
	uint32_t L = BITS(inst, 22, 22);
	uint32_t Rn = BITS(inst, 9, 5);
	uint32_t fields = (opc << 2) | (V << 1) | L;
	uint8_t scale = 2 + (opc >> 1);

	if (__builtin_expect((4 == fields) ||
	    (12 <= fields), false)) {
		CANNOTDECODE("invalid", inst);
		return false;
	}

	if (V == 1) {
		scale = 2 + opc;
	} else {
		scale = 2 + BITS(opc, 1, 1);
	}

	info->bytes = 2 * (1 << scale);
	// post-indexed
	info->addr = ss->ss_64.x[Rn];

	return true;
}

// load/store register pair (pre-indexed)
static bool
get_info_c3316(uint32_t inst, arm_saved_state_t *ss, instruction_info_t *info)
{
	uint32_t opc = BITS(inst, 31, 30);
	uint32_t V = BITS(inst, 26, 26);
	uint32_t L = BITS(inst, 22, 22);
	uint32_t imm7 = BITS(inst, 21, 15);
	uint32_t Rn = BITS(inst, 9, 5);
	uint32_t fields = (opc << 2) | (V << 1) | L;
	uint8_t scale = 2 + (opc >> 1);

	if (__builtin_expect((4 == fields) ||
	    (12 <= fields), false)) {
		CANNOTDECODE("invalid", inst);
		return false;
	}

	if (V == 1) {
		scale = 2 + opc;
	} else {
		scale = 2 + BITS(opc, 1, 1);
	}

	info->bytes = 2 * (1 << scale);
	info->addr = ss->ss_64.x[Rn] + (SIGN_EXTEND_64(imm7, 7) << scale);

	return true;
}


//-------------------------------------------------------------------
// Globals
//
int
pgtrace_decode_and_run(uint32_t inst, vm_offset_t fva, vm_map_offset_t *cva_page, arm_saved_state_t *ss, pgtrace_run_result_t *res)
{
	uint8_t len = sizeof(typetbl) / sizeof(type_entry_t);
	run_t run = NULL;
	get_info_t get_info = NULL;
	vm_offset_t pa, cva;
	vm_offset_t cva_front_page = cva_page[0];
	vm_offset_t cva_cur_page = cva_page[1];
	instruction_info_t info;

	for (uint8_t i = 0; i < len; i++) {
		if ((typetbl[i].mask & inst) == typetbl[i].value) {
			run = typetbl[i].run;
			get_info = typetbl[i].get_info;
			break;
		}
	}

	assert(run != NULL && get_info != NULL);

	get_info(inst, ss, &info);

	if (info.addr == fva) {
		cva = cva_cur_page + (fva & ARM_PGMASK);
	} else {
		// which means a front page is not a tracing page
		cva = cva_front_page + (fva & ARM_PGMASK);
	}

	pa = mmu_kvtop(cva);
	if (!pa) {
		panic("%s: invalid address cva=%lx fva=%lx info.addr=%lx inst=%x", __func__, cva, fva, info.addr, inst);
	}

	absolutetime_to_nanoseconds(mach_absolute_time(), &res->rr_time);
	run(inst, pa, cva, ss, res);

	return 0;
}

void
pgtrace_decoder_get_stats(pgtrace_stats_t *s)
{
	memcpy((void *)&(s->stat_decoder), &(stats.stat_decoder), sizeof(stats.stat_decoder));
}
#endif