[ust.git] / include / ust / processor.h

/* Copyright (C) 2009  Pierre-Marc Fournier
 *
 * This library is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * This library is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with this library; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301 USA
 */

#ifndef UST_PROCESSOR_H
#define UST_PROCESSOR_H

#include <stddef.h>
#include <string.h>
#include <urcu-bp.h>

extern __thread long ust_reg_stack[500];
extern volatile __thread long *ust_reg_stack_ptr;

#define ____cacheline_aligned __attribute__((aligned(CAA_CACHE_LINE_SIZE)))

#ifdef __i386

struct registers {
	short ss;
	short cs;
	long esi;
	long ebp;
	long edx;
	long edi;
	long ecx;
	long ebx;
	long eax;
	long eflags;
	long esp;
};

static inline int fls(int x)
{
        int r;
        asm("bsrl %1,%0\n\t"
            "cmovzl %2,%0"
            : "=&r" (r) : "rm" (x), "rm" (-1));
        return r + 1;
}

#ifdef CONFIG_UST_GDB_INTEGRATION

/* save_registers - saves most of the processor's registers so
 * they are available to the probe. gdb uses this to give the
 * value of local variables.
 *
 * Saving all registers without losing any of their values is
 * tricky.
 *
 * We cannot pass to the asm stub the address of a registers structure
 * on the stack, because it will use a register and override its value.
 *
 * We don't want to use a stub to push the regs on the stack and then
 * another stub to copy them to a structure because changing %sp in asm
 * and then returning to C (even briefly) can have unexpected results.
 * Also, gcc might modify %sp between the stubs in reaction to the
 * register needs of the second stub that needs to know where to copy
 * the register values.
 *
 * So the chosen approach is to use another stack, declared in thread-
 * local storage, to push the registers. They are subsequently copied
 * to the stack, by C code.
 */

#define save_registers(regsptr) \
	asm volatile ( \
	     /* save original esp */ \
	     "pushl %%esp\n\t" \
	     /* push original eflags */ \
	     "pushfl\n\t" \
	      /* eax will hold the ptr to the private stack bottom */ \
	     "pushl %%eax\n\t" \
	     /* ebx is used for TLS access */ \
	     "pushl %%ebx\n\t" \
	     /* ecx will be used to temporarily hold the stack bottom addr */\
	     "pushl %%ecx\n\t"						     \
	     /* rdi is the input to __tls_get_addr, and also a temp var */   \
	     "pushl %%edi\n\t"						     \
	     /* For TLS access, we have to do function calls. However,	     \
	      * we must not lose the original value of:			     \
	      *   esp, eflags, eax, ebx, ecx, edx, esi, edi, ebp, cs, ss     \
	      *								     \
	      * Some registers' original values have already been saved:     \
	      *   esp, eflags, eax, ebx, ecx, edi			     \
	      *								     \
	      * In addition, the i386 ABI says the following registers belong\
	      * to the caller function:					     \
	      *   esp, ebp, esi, edi, ebx				     \
	      *								     \
	      * The following registers should not be changed by the callee: \
	      *   cs, ss						     \
	      *								     \
	      * Therefore, the following registers must be explicitly 	     \
	      * preserved:						     \
	      *   edx							     \
	      */ \
	     "pushl %%edx\n\t" \
	     /* Get GOT address */ \
	     "call __i686.get_pc_thunk.bx\n\t" \
	     "addl $_GLOBAL_OFFSET_TABLE_, %%ebx\n\t" \
	     /* Start TLS access of private reg stack pointer */ \
	     "leal ust_reg_stack_ptr@tlsgd(,%%ebx,1),%%eax\n\t" \
	     "call ___tls_get_addr@plt\n\t" \
	     /* --- End TLS access */ \
	     /* check if ust_reg_stack_ptr has been initialized */ \
	     "movl (%%eax),%%ecx\n\t" \
	     "testl %%ecx,%%ecx\n\t" \
	     "jne 1f\n\t" \
	     "movl %%eax,%%ecx\n\t" \
	     /* Save ecx because we are using it. */ \
	     "pushl %%ecx\n\t" \
	     /* Start TLS access of private reg stack */ \
	     "leal ust_reg_stack@tlsgd(,%%ebx,1),%%eax\n\t" \
	     "call ___tls_get_addr@plt\n\t" \
	     /* --- End TLS access */ \
	     "popl %%ecx\n\t" \
	     "addl $500,%%eax\n\t" \
	     "movl %%eax,(%%ecx)\n\t" \
	     "movl %%ecx,%%eax\n\t" \
	     /* now the pointer to the private stack is in eax. \
	        must add stack size so the ptr points to the stack bottom. */ \
	"1:\n\t" \
	     /* edx was pushed for function calls */ \
	     "popl %%edx\n\t" \
	     /* Manually push esp to private stack */ \
	     "addl $-4,(%%eax)\n\t" \
	     "movl 20(%%esp), %%edi\n\t" \
	     "movl (%%eax), %%ebx\n\t" \
	     "movl %%edi, (%%ebx)\n\t" \
	     /* Manually push eflags to private stack */ \
	     "addl $-4,(%%eax)\n\t" \
	     "movl 16(%%esp), %%edi\n\t" \
	     "movl (%%eax), %%ebx\n\t" \
	     "movl %%edi, (%%ebx)\n\t" \
	     /* Manually push eax to private stack */ \
	     "addl $-4,(%%eax)\n\t" \
	     "movl 12(%%esp), %%edi\n\t" \
	     "movl (%%eax), %%ebx\n\t" \
	     "movl %%edi, (%%ebx)\n\t" \
	     /* Manually push ebx to private stack */ \
	     "addl $-4,(%%eax)\n\t" \
	     "movl 8(%%esp), %%edi\n\t" \
	     "movl (%%eax), %%ebx\n\t" \
	     "movl %%edi, (%%ebx)\n\t" \
	     /* Manually push ecx to private stack */ \
	     "addl $-4,(%%eax)\n\t" \
	     "movl 4(%%esp), %%edi\n\t" \
	     "movl (%%eax), %%ebx\n\t" \
	     "movl %%edi, (%%ebx)\n\t" \
	     /* Manually push edi to private stack */ \
	     "addl $-4,(%%eax)\n\t" \
	     "movl 0(%%esp), %%edi\n\t" \
	     "movl (%%eax), %%ebx\n\t" \
	     "movl %%edi, (%%ebx)\n\t" \
	     /* now push regs to tls */ \
	     /* -- esp already pushed -- */ \
	     /* -- eax already pushed -- */ \
	     /* -- ebx already pushed -- */ \
	     /* -- ecx already pushed -- */ \
	     /* -- edi already pushed -- */ \
	     "addl $-4,(%%eax)\n\t" \
	     "movl (%%eax), %%ebx\n\t" \
	     "movl %%edx,(%%ebx)\n\t" \
	     "addl $-4,(%%eax)\n\t" \
	     "movl (%%eax), %%ebx\n\t" \
	     "movl %%ebp,(%%ebx)\n\t" \
	     "addl $-4,(%%eax)\n\t" \
	     "movl (%%eax), %%ebx\n\t" \
	     "movl %%esi,(%%ebx)\n\t" \
	     /* push cs */ \
	     "addl $-2,(%%eax)\n\t" \
	     "movl (%%eax), %%ebx\n\t" \
	     "movw %%cs, (%%ebx)\n\t" \
	     /* push ss */ \
	     "addl $-2,(%%eax)\n\t" \
	     "movl (%%eax), %%ebx\n\t" \
	     "movw %%ss, (%%ebx)\n\t" \
	     /* restore original values of regs that were used internally */ \
	     "popl %%edi\n\t" \
	     "popl %%ecx\n\t" \
	     "popl %%ebx\n\t" \
	     "popl %%eax\n\t" \
	     /* cancel push of rsp */ \
	     "addl $4,%%esp\n\t" \
	     /* cancel push of eflags */ \
	     "addl $4,%%esp\n\t" \
	     ::: "memory"); \
	memcpy(regsptr, (void *)ust_reg_stack_ptr, sizeof(struct registers)); \
	ust_reg_stack_ptr = (void *)(((long)ust_reg_stack_ptr) + sizeof(struct registers));

#else /* CONFIG_UST_GDB_INTEGRATION */

#define save_registers(a)

#endif /* CONFIG_UST_GDB_INTEGRATION */

#define RELATIVE_ADDRESS(__rel_label__) __rel_label__

#define ARCH_COPY_ADDR(dst) "lea 2b," dst "\n\t"

#define _ASM_PTR ".long "

#endif /* below is code for x86-64 */

#ifdef __x86_64

struct registers {
	int padding; /* 4 bytes */
	short ss;
	short cs;
	unsigned long r15;
	unsigned long r14;
	unsigned long r13;
	unsigned long r12;
	unsigned long r11;
	unsigned long r10;
	unsigned long r9;
	unsigned long r8;
	unsigned long rsi;
	unsigned long rbp;
	unsigned long rdx;
	unsigned long rcx;
	unsigned long rdi;
	unsigned long rbx;
	unsigned long rax;
	unsigned long rflags;
	unsigned long rsp;
};

static inline int fls(int x)
{
        int r;
        asm("bsrl %1,%0\n\t"
            "cmovzl %2,%0"
            : "=&r" (r) : "rm" (x), "rm" (-1));
        return r + 1;
}

#ifdef CONFIG_UST_GDB_INTEGRATION

#define save_registers(regsptr) \
	asm volatile ( \
	     /* save original rsp */ \
	     "pushq %%rsp\n\t" \
	     /* push original rflags */ \
	     "pushfq\n\t" \
	      /* rax will hold the ptr to the private stack bottom */ \
	     "pushq %%rax\n\t" \
	     /* rbx will be used to temporarily hold the stack bottom addr */ \
	     "pushq %%rbx\n\t" \
	     /* rdi is the input to __tls_get_addr, and also a temp var */ \
	     "pushq %%rdi\n\t" \
	     /* For TLS access, we have to do function calls. However,	     \
	      * we must not lose the original value of:			     \
	      *   rsp, rflags, rax, rbx, rcx, rdx, rsi, rdi, rbp, r8, r9     \
	      *   r10, r11, r12, r13, r14, r15, cs, ss			     \
	      *								     \
	      * Some registers' original values have already been saved:     \
	      *   rsp, rflags, rax, rbx, rdi				     \
	      *								     \
	      * In addition, the x86-64 ABI says the following registers     \
	      * belong to the caller function:				     \
	      *   rbp, rbx, r12, r13, r14, r15				     \
	      *								     \
	      * The following registers should not be changed by the callee: \
	      *   cs, ss						     \
	      *								     \
	      * Therefore, the following registers must be explicitly 	     \
	      * preserved:						     \
	      *   rcx, rdx, rsi, r8, r9, r10, r11			     \
	      */ \
	     "pushq %%rcx\n\t" \
	     "pushq %%rdx\n\t" \
	     "pushq %%rsi\n\t" \
	     "pushq %%r8\n\t" \
	     "pushq %%r9\n\t" \
	     "pushq %%r10\n\t" \
	     "pushq %%r11\n\t" \
	     /* Start TLS access of private reg stack pointer */ \
	     ".byte 0x66\n\t" \
	     "leaq ust_reg_stack_ptr@tlsgd(%%rip), %%rdi\n\t" \
	     ".word 0x6666\n\t" \
	     "rex64\n\t" \
	     "call __tls_get_addr@plt\n\t" \
	     /* --- End TLS access */ \
	     /* check if ust_reg_stack_ptr has been initialized */ \
	     "movq (%%rax),%%rbx\n\t" \
	     "testq %%rbx,%%rbx\n\t" \
	     "jne 1f\n\t" \
	     "movq %%rax,%%rbx\n\t" \
	     /* Start TLS access of private reg stack */ \
	     ".byte 0x66\n\t" \
	     "leaq ust_reg_stack@tlsgd(%%rip), %%rdi\n\t" \
	     ".word 0x6666\n\t" \
	     "rex64\n\t" \
	     "call __tls_get_addr@plt\n\t" \
	     /* --- End TLS access */ \
	     "addq $500,%%rax\n\t" \
	     "movq %%rax,(%%rbx)\n\t" \
	     "movq %%rbx,%%rax\n\t" \
	     /* now the pointer to the private stack is in rax.
	        must add stack size so the ptr points to the stack bottom. */ \
	"1:\n\t" \
	     /* Pop regs that were pushed for function calls */ \
	     "popq %%r11\n\t" \
	     "popq %%r10\n\t" \
	     "popq %%r9\n\t" \
	     "popq %%r8\n\t" \
	     "popq %%rsi\n\t" \
	     "popq %%rdx\n\t" \
	     "popq %%rcx\n\t" \
	     /* Manually push rsp to private stack */ \
	     "addq $-8,(%%rax)\n\t" \
	     "movq 32(%%rsp), %%rdi\n\t" \
	     "movq (%%rax), %%rbx\n\t" \
	     "movq %%rdi, (%%rbx)\n\t" \
	     /* Manually push eflags to private stack */ \
	     "addq $-8,(%%rax)\n\t" \
	     "movq 24(%%rsp), %%rdi\n\t" \
	     "movq (%%rax), %%rbx\n\t" \
	     "movq %%rdi, (%%rbx)\n\t" \
	     /* Manually push rax to private stack */ \
	     "addq $-8,(%%rax)\n\t" \
	     "movq 16(%%rsp), %%rdi\n\t" \
	     "movq (%%rax), %%rbx\n\t" \
	     "movq %%rdi, (%%rbx)\n\t" \
	     /* Manually push rbx to private stack */ \
	     "addq $-8,(%%rax)\n\t" \
	     "movq 8(%%rsp), %%rdi\n\t" \
	     "movq (%%rax), %%rbx\n\t" \
	     "movq %%rdi, (%%rbx)\n\t" \
	     /* Manually push rdi to private stack */ \
	     "addq $-8,(%%rax)\n\t" \
	     "movq 0(%%rsp), %%rdi\n\t" \
	     "movq (%%rax), %%rbx\n\t" \
	     "movq %%rdi, (%%rbx)\n\t" \
	     /* now push regs to tls */ \
	     /* -- rsp already pushed -- */ \
	     /* -- rax already pushed -- */ \
	     /* -- rbx already pushed -- */ \
	     /* -- rdi already pushed -- */ \
	     "addq $-8,(%%rax)\n\t" \
	     "movq (%%rax), %%rbx\n\t" \
	     "movq %%rcx,(%%rbx)\n\t" \
	     "addq $-8,(%%rax)\n\t" \
	     "movq (%%rax), %%rbx\n\t" \
	     "movq %%rdx,(%%rbx)\n\t" \
	     "addq $-8,(%%rax)\n\t" \
	     "movq (%%rax), %%rbx\n\t" \
	     "movq %%rbp,(%%rbx)\n\t" \
	     "addq $-8,(%%rax)\n\t" \
	     "movq (%%rax), %%rbx\n\t" \
	     "movq %%rsi,(%%rbx)\n\t" \
	     "addq $-8,(%%rax)\n\t" \
	     "movq (%%rax), %%rbx\n\t" \
	     "movq %%r8,(%%rbx)\n\t" \
	     "addq $-8,(%%rax)\n\t" \
	     "movq (%%rax), %%rbx\n\t" \
	     "movq %%r9,(%%rbx)\n\t" \
	     "addq $-8,(%%rax)\n\t" \
	     "movq (%%rax), %%rbx\n\t" \
	     "movq %%r10,(%%rbx)\n\t" \
	     "addq $-8,(%%rax)\n\t" \
	     "movq (%%rax), %%rbx\n\t" \
	     "movq %%r11,(%%rbx)\n\t" \
	     "addq $-8,(%%rax)\n\t" \
	     "movq (%%rax), %%rbx\n\t" \
	     "movq %%r12,(%%rbx)\n\t" \
	     "addq $-8,(%%rax)\n\t" \
	     "movq (%%rax), %%rbx\n\t" \
	     "movq %%r13,(%%rbx)\n\t" \
	     "addq $-8,(%%rax)\n\t" \
	     "movq (%%rax), %%rbx\n\t" \
	     "movq %%r14,(%%rbx)\n\t" \
	     "addq $-8,(%%rax)\n\t" \
	     "movq (%%rax), %%rbx\n\t" \
	     "movq %%r15,(%%rbx)\n\t" \
	     /* push cs */ \
	     "addq $-2,(%%rax)\n\t" \
	     "movq (%%rax), %%rbx\n\t" \
	     "movw %%cs, (%%rbx)\n\t" \
	     /* push ss */ \
	     "addq $-2,(%%rax)\n\t" \
	     "movq (%%rax), %%rbx\n\t" \
	     "movw %%ss, (%%rbx)\n\t" \
	     /* add padding for struct registers */ \
	     "addq $-4,(%%rax)\n\t" \
	     /* restore original values of regs that were used internally */ \
	     "popq %%rdi\n\t" \
	     "popq %%rbx\n\t" \
	     "popq %%rax\n\t" \
	     /* cancel push of rsp */ \
	     "addq $8,%%rsp\n\t" \
	     /* cancel push of rflags */ \
	     "addq $8,%%rsp\n\t" \
	     ::); \
	memcpy(regsptr, (void *)ust_reg_stack_ptr, sizeof(struct registers)); \
	ust_reg_stack_ptr = (void *)(((long)ust_reg_stack_ptr) + sizeof(struct registers));

#else /* CONFIG_UST_GDB_INTEGRATION */

#define save_registers(a)

#endif /* CONFIG_UST_GDB_INTEGRATION */

/* Macro to insert the address of a relative jump in an assembly stub,
 * in a relocatable way. On x86-64, this uses a special (%rip) notation. */
#define RELATIVE_ADDRESS(__rel_label__) __rel_label__(%%rip)

#define ARCH_COPY_ADDR(dst) "lea 2b(%%rip)," dst "\n\t"

#define _ASM_PTR ".quad "

#endif /* x86_64 */

#ifdef __PPC__

struct registers {
};

static __inline__ int fls(unsigned int x)
{
        int lz;

        asm ("cntlzw %0,%1" : "=r" (lz) : "r" (x));
        return 32 - lz;
}

#define ARCH_COPY_ADDR(dst) \
		     "lis " dst ",2b@h\n\t" /* load high bytes */ \
		     "ori " dst "," dst ",2b@l\n\t" /* load low bytes */

#define _ASM_PTR ".long "
#define save_registers(a)

#endif /* __PPC__ */

#endif /* UST_PROCESSOR_H */
Commit	Line	Data
a09dac63 PMF	1	/* Copyright (C) 2009 Pierre-Marc Fournier
	2	*
	3	* This library is free software; you can redistribute it and/or
	4	* modify it under the terms of the GNU Lesser General Public
	5	* License as published by the Free Software Foundation; either
	6	* version 2.1 of the License, or (at your option) any later version.
	7	*
	8	* This library is distributed in the hope that it will be useful,
	9	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	10	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	11	* Lesser General Public License for more details.
	12	*
	13	* You should have received a copy of the GNU Lesser General Public
	14	* License along with this library; if not, write to the Free Software
	15	* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
	16	*/
	17
5af57e62 PMF	18	#ifndef UST_PROCESSOR_H
5af57e62 PMF	19	#define UST_PROCESSOR_H
d98a01c6 PMF	20
d98a01c6 PMF	21	#include <stddef.h>
636ca5d6	22	#include <string.h>
066b83b8	23	#include <urcu-bp.h>
636ca5d6 PMF	24
	25	extern __thread long ust_reg_stack[500];
	26	extern volatile __thread long *ust_reg_stack_ptr;
	27
0222e121	28	#define ____cacheline_aligned __attribute__((aligned(CAA_CACHE_LINE_SIZE)))
066b83b8	29
6fa0d97f	30	#ifdef __i386
d98a01c6 PMF	31
d98a01c6 PMF	32	struct registers {
7756d65a PMF	33	short ss;
7756d65a PMF	34	short cs;
d98a01c6	35	long esi;
7756d65a PMF	36	long ebp;
7756d65a PMF	37	long edx;
d98a01c6	38	long edi;
f2496f58	39	long ecx;
7756d65a PMF	40	long ebx;
7756d65a PMF	41	long eax;
d98a01c6	42	long eflags;
7756d65a	43	long esp;
d98a01c6 PMF	44	};
d98a01c6 PMF	45
6fa0d97f PMF	46	static inline int fls(int x)
	47	{
	48	int r;
	49	asm("bsrl %1,%0\n\t"
	50	"cmovzl %2,%0"
	51	: "=&r" (r) : "rm" (x), "rm" (-1));
	52	return r + 1;
	53	}
	54
e003d6ee	55	#ifdef CONFIG_UST_GDB_INTEGRATION
defa46a7	56
55c5b393 PMF	57	/* save_registers - saves most of the processor's registers so
	58	* they are available to the probe. gdb uses this to give the
	59	* value of local variables.
	60	*
	61	* Saving all registers without losing any of their values is
	62	* tricky.
	63	*
	64	* We cannot pass to the asm stub the address of a registers structure
	65	* on the stack, because it will use a register and override its value.
	66	*
	67	* We don't want to use a stub to push the regs on the stack and then
	68	* another stub to copy them to a structure because changing %sp in asm
	69	* and then returning to C (even briefly) can have unexpected results.
	70	* Also, gcc might modify %sp between the stubs in reaction to the
	71	* register needs of the second stub that needs to know where to copy
	72	* the register values.
	73	*
	74	* So the chosen approach is to use another stack, declared in thread-
	75	* local storage, to push the registers. They are subsequently copied
	76	* to the stack, by C code.
	77	*/
7756d65a PMF	78
	79	#define save_registers(regsptr) \
	80	asm volatile ( \
	81	/* save original esp */ \
	82	"pushl %%esp\n\t" \
	83	/* push original eflags */ \
	84	"pushfl\n\t" \
	85	/* eax will hold the ptr to the private stack bottom */ \
	86	"pushl %%eax\n\t" \
f2496f58	87	/* ebx is used for TLS access */ \
7756d65a	88	"pushl %%ebx\n\t" \
55c5b393 PMF	89	/* ecx will be used to temporarily hold the stack bottom addr */\
	90	"pushl %%ecx\n\t" \
	91	/* rdi is the input to __tls_get_addr, and also a temp var */ \
	92	"pushl %%edi\n\t" \
	93	/* For TLS access, we have to do function calls. However, \
	94	* we must not lose the original value of: \
	95	* esp, eflags, eax, ebx, ecx, edx, esi, edi, ebp, cs, ss \
	96	* \
	97	* Some registers' original values have already been saved: \
	98	* esp, eflags, eax, ebx, ecx, edi \
	99	* \
	100	* In addition, the i386 ABI says the following registers belong\
	101	* to the caller function: \
	102	* esp, ebp, esi, edi, ebx \
	103	* \
	104	* The following registers should not be changed by the callee: \
	105	* cs, ss \
	106	* \
	107	* Therefore, the following registers must be explicitly \
	108	* preserved: \
	109	* edx \
	110	*/ \
	111	"pushl %%edx\n\t" \
f2496f58 PMF	112	/* Get GOT address */ \
	113	"call __i686.get_pc_thunk.bx\n\t" \
	114	"addl $_GLOBAL_OFFSET_TABLE_, %%ebx\n\t" \
7756d65a PMF	115	/* Start TLS access of private reg stack pointer */ \
	116	"leal ust_reg_stack_ptr@tlsgd(,%%ebx,1),%%eax\n\t" \
	117	"call ___tls_get_addr@plt\n\t" \
	118	/* --- End TLS access */ \
	119	/* check if ust_reg_stack_ptr has been initialized */ \
f2496f58 PMF	120	"movl (%%eax),%%ecx\n\t" \
f2496f58 PMF	121	"testl %%ecx,%%ecx\n\t" \
7756d65a	122	"jne 1f\n\t" \
f2496f58	123	"movl %%eax,%%ecx\n\t" \
55c5b393	124	/* Save ecx because we are using it. */ \
f2496f58	125	"pushl %%ecx\n\t" \
7756d65a PMF	126	/* Start TLS access of private reg stack */ \
	127	"leal ust_reg_stack@tlsgd(,%%ebx,1),%%eax\n\t" \
	128	"call ___tls_get_addr@plt\n\t" \
	129	/* --- End TLS access */ \
f2496f58	130	"popl %%ecx\n\t" \
7756d65a	131	"addl $500,%%eax\n\t" \
f2496f58 PMF	132	"movl %%eax,(%%ecx)\n\t" \
f2496f58 PMF	133	"movl %%ecx,%%eax\n\t" \
7756d65a PMF	134	/* now the pointer to the private stack is in eax. \
	135	must add stack size so the ptr points to the stack bottom. */ \
	136	"1:\n\t" \
55c5b393 PMF	137	/* edx was pushed for function calls */ \
55c5b393 PMF	138	"popl %%edx\n\t" \
7756d65a PMF	139	/* Manually push esp to private stack */ \
7756d65a PMF	140	"addl $-4,(%%eax)\n\t" \
f2496f58	141	"movl 20(%%esp), %%edi\n\t" \
7756d65a PMF	142	"movl (%%eax), %%ebx\n\t" \
	143	"movl %%edi, (%%ebx)\n\t" \
	144	/* Manually push eflags to private stack */ \
	145	"addl $-4,(%%eax)\n\t" \
f2496f58	146	"movl 16(%%esp), %%edi\n\t" \
7756d65a PMF	147	"movl (%%eax), %%ebx\n\t" \
	148	"movl %%edi, (%%ebx)\n\t" \
	149	/* Manually push eax to private stack */ \
	150	"addl $-4,(%%eax)\n\t" \
f2496f58	151	"movl 12(%%esp), %%edi\n\t" \
7756d65a PMF	152	"movl (%%eax), %%ebx\n\t" \
	153	"movl %%edi, (%%ebx)\n\t" \
	154	/* Manually push ebx to private stack */ \
	155	"addl $-4,(%%eax)\n\t" \
f2496f58 PMF	156	"movl 8(%%esp), %%edi\n\t" \
	157	"movl (%%eax), %%ebx\n\t" \
	158	"movl %%edi, (%%ebx)\n\t" \
	159	/* Manually push ecx to private stack */ \
	160	"addl $-4,(%%eax)\n\t" \
7756d65a PMF	161	"movl 4(%%esp), %%edi\n\t" \
	162	"movl (%%eax), %%ebx\n\t" \
	163	"movl %%edi, (%%ebx)\n\t" \
	164	/* Manually push edi to private stack */ \
	165	"addl $-4,(%%eax)\n\t" \
	166	"movl 0(%%esp), %%edi\n\t" \
	167	"movl (%%eax), %%ebx\n\t" \
	168	"movl %%edi, (%%ebx)\n\t" \
	169	/* now push regs to tls */ \
	170	/* -- esp already pushed -- */ \
	171	/* -- eax already pushed -- */ \
	172	/* -- ebx already pushed -- */ \
f2496f58	173	/* -- ecx already pushed -- */ \
7756d65a PMF	174	/* -- edi already pushed -- */ \
	175	"addl $-4,(%%eax)\n\t" \
	176	"movl (%%eax), %%ebx\n\t" \
7756d65a PMF	177	"movl %%edx,(%%ebx)\n\t" \
	178	"addl $-4,(%%eax)\n\t" \
	179	"movl (%%eax), %%ebx\n\t" \
	180	"movl %%ebp,(%%ebx)\n\t" \
	181	"addl $-4,(%%eax)\n\t" \
	182	"movl (%%eax), %%ebx\n\t" \
	183	"movl %%esi,(%%ebx)\n\t" \
	184	/* push cs */ \
	185	"addl $-2,(%%eax)\n\t" \
	186	"movl (%%eax), %%ebx\n\t" \
	187	"movw %%cs, (%%ebx)\n\t" \
	188	/* push ss */ \
	189	"addl $-2,(%%eax)\n\t" \
	190	"movl (%%eax), %%ebx\n\t" \
	191	"movw %%ss, (%%ebx)\n\t" \
	192	/* restore original values of regs that were used internally */ \
	193	"popl %%edi\n\t" \
f2496f58	194	"popl %%ecx\n\t" \
7756d65a PMF	195	"popl %%ebx\n\t" \
	196	"popl %%eax\n\t" \
	197	/* cancel push of rsp */ \
	198	"addl $4,%%esp\n\t" \
	199	/* cancel push of eflags */ \
	200	"addl $4,%%esp\n\t" \
	201	::: "memory"); \
	202	memcpy(regsptr, (void *)ust_reg_stack_ptr, sizeof(struct registers)); \
	203	ust_reg_stack_ptr = (void *)(((long)ust_reg_stack_ptr) + sizeof(struct registers));
defa46a7	204
e003d6ee	205	#else /* CONFIG_UST_GDB_INTEGRATION */
defa46a7	206
defa46a7 PMF	207	#define save_registers(a)
defa46a7 PMF	208
e003d6ee	209	#endif /* CONFIG_UST_GDB_INTEGRATION */
d98a01c6	210
9e8f4f52 PMF	211	#define RELATIVE_ADDRESS(__rel_label__) __rel_label__
9e8f4f52 PMF	212
6fa0d97f	213	#define ARCH_COPY_ADDR(dst) "lea 2b," dst "\n\t"
9692fa18	214
9e8f4f52 PMF	215	#define _ASM_PTR ".long "
9e8f4f52 PMF	216
6fa0d97f PMF	217	#endif /* below is code for x86-64 */
	218
	219	#ifdef __x86_64
d98a01c6 PMF	220
d98a01c6 PMF	221	struct registers {
636ca5d6 PMF	222	int padding; /* 4 bytes */
	223	short ss;
	224	short cs;
636ca5d6 PMF	225	unsigned long r15;
	226	unsigned long r14;
	227	unsigned long r13;
	228	unsigned long r12;
	229	unsigned long r11;
	230	unsigned long r10;
	231	unsigned long r9;
	232	unsigned long r8;
d98a01c6	233	unsigned long rsi;
636ca5d6 PMF	234	unsigned long rbp;
	235	unsigned long rdx;
	236	unsigned long rcx;
d98a01c6	237	unsigned long rdi;
636ca5d6 PMF	238	unsigned long rbx;
636ca5d6 PMF	239	unsigned long rax;
a5850bc4	240	unsigned long rflags;
636ca5d6	241	unsigned long rsp;
d98a01c6 PMF	242	};
d98a01c6 PMF	243
6fa0d97f PMF	244	static inline int fls(int x)
	245	{
	246	int r;
	247	asm("bsrl %1,%0\n\t"
	248	"cmovzl %2,%0"
	249	: "=&r" (r) : "rm" (x), "rm" (-1));
	250	return r + 1;
	251	}
	252
e003d6ee	253	#ifdef CONFIG_UST_GDB_INTEGRATION
defa46a7	254
8524c98d	255	#define save_registers(regsptr) \
636ca5d6 PMF	256	asm volatile ( \
	257	/* save original rsp */ \
	258	"pushq %%rsp\n\t" \
a5850bc4 PMF	259	/* push original rflags */ \
a5850bc4 PMF	260	"pushfq\n\t" \
636ca5d6 PMF	261	/* rax will hold the ptr to the private stack bottom */ \
	262	"pushq %%rax\n\t" \
	263	/* rbx will be used to temporarily hold the stack bottom addr */ \
	264	"pushq %%rbx\n\t" \
	265	/* rdi is the input to __tls_get_addr, and also a temp var */ \
	266	"pushq %%rdi\n\t" \
55c5b393 PMF	267	/* For TLS access, we have to do function calls. However, \
	268	* we must not lose the original value of: \
	269	* rsp, rflags, rax, rbx, rcx, rdx, rsi, rdi, rbp, r8, r9 \
	270	* r10, r11, r12, r13, r14, r15, cs, ss \
	271	* \
	272	* Some registers' original values have already been saved: \
	273	* rsp, rflags, rax, rbx, rdi \
	274	* \
	275	* In addition, the x86-64 ABI says the following registers \
	276	* belong to the caller function: \
	277	* rbp, rbx, r12, r13, r14, r15 \
	278	* \
	279	* The following registers should not be changed by the callee: \
	280	* cs, ss \
	281	* \
	282	* Therefore, the following registers must be explicitly \
	283	* preserved: \
	284	* rcx, rdx, rsi, r8, r9, r10, r11 \
	285	*/ \
	286	"pushq %%rcx\n\t" \
	287	"pushq %%rdx\n\t" \
	288	"pushq %%rsi\n\t" \
	289	"pushq %%r8\n\t" \
	290	"pushq %%r9\n\t" \
	291	"pushq %%r10\n\t" \
	292	"pushq %%r11\n\t" \
a5850bc4	293	/* Start TLS access of private reg stack pointer */ \
636ca5d6 PMF	294	".byte 0x66\n\t" \
	295	"leaq ust_reg_stack_ptr@tlsgd(%%rip), %%rdi\n\t" \
	296	".word 0x6666\n\t" \
	297	"rex64\n\t" \
	298	"call __tls_get_addr@plt\n\t" \
	299	/* --- End TLS access */ \
a5850bc4 PMF	300	/* check if ust_reg_stack_ptr has been initialized */ \
	301	"movq (%%rax),%%rbx\n\t" \
	302	"testq %%rbx,%%rbx\n\t" \
	303	"jne 1f\n\t" \
	304	"movq %%rax,%%rbx\n\t" \
	305	/* Start TLS access of private reg stack */ \
	306	".byte 0x66\n\t" \
	307	"leaq ust_reg_stack@tlsgd(%%rip), %%rdi\n\t" \
	308	".word 0x6666\n\t" \
	309	"rex64\n\t" \
	310	"call __tls_get_addr@plt\n\t" \
	311	/* --- End TLS access */ \
	312	"addq $500,%%rax\n\t" \
	313	"movq %%rax,(%%rbx)\n\t" \
	314	"movq %%rbx,%%rax\n\t" \
	315	/* now the pointer to the private stack is in rax.
	316	must add stack size so the ptr points to the stack bottom. */ \
	317	"1:\n\t" \
55c5b393 PMF	318	/* Pop regs that were pushed for function calls */ \
	319	"popq %%r11\n\t" \
	320	"popq %%r10\n\t" \
	321	"popq %%r9\n\t" \
	322	"popq %%r8\n\t" \
	323	"popq %%rsi\n\t" \
	324	"popq %%rdx\n\t" \
	325	"popq %%rcx\n\t" \
636ca5d6 PMF	326	/* Manually push rsp to private stack */ \
636ca5d6 PMF	327	"addq $-8,(%%rax)\n\t" \
a5850bc4 PMF	328	"movq 32(%%rsp), %%rdi\n\t" \
	329	"movq (%%rax), %%rbx\n\t" \
	330	"movq %%rdi, (%%rbx)\n\t" \
	331	/* Manually push eflags to private stack */ \
	332	"addq $-8,(%%rax)\n\t" \
636ca5d6 PMF	333	"movq 24(%%rsp), %%rdi\n\t" \
	334	"movq (%%rax), %%rbx\n\t" \
	335	"movq %%rdi, (%%rbx)\n\t" \
	336	/* Manually push rax to private stack */ \
	337	"addq $-8,(%%rax)\n\t" \
	338	"movq 16(%%rsp), %%rdi\n\t" \
	339	"movq (%%rax), %%rbx\n\t" \
	340	"movq %%rdi, (%%rbx)\n\t" \
	341	/* Manually push rbx to private stack */ \
	342	"addq $-8,(%%rax)\n\t" \
	343	"movq 8(%%rsp), %%rdi\n\t" \
	344	"movq (%%rax), %%rbx\n\t" \
	345	"movq %%rdi, (%%rbx)\n\t" \
	346	/* Manually push rdi to private stack */ \
	347	"addq $-8,(%%rax)\n\t" \
	348	"movq 0(%%rsp), %%rdi\n\t" \
	349	"movq (%%rax), %%rbx\n\t" \
	350	"movq %%rdi, (%%rbx)\n\t" \
	351	/* now push regs to tls */ \
	352	/* -- rsp already pushed -- */ \
	353	/* -- rax already pushed -- */ \
	354	/* -- rbx already pushed -- */ \
	355	/* -- rdi already pushed -- */ \
	356	"addq $-8,(%%rax)\n\t" \
	357	"movq (%%rax), %%rbx\n\t" \
	358	"movq %%rcx,(%%rbx)\n\t" \
	359	"addq $-8,(%%rax)\n\t" \
	360	"movq (%%rax), %%rbx\n\t" \
	361	"movq %%rdx,(%%rbx)\n\t" \
	362	"addq $-8,(%%rax)\n\t" \
	363	"movq (%%rax), %%rbx\n\t" \
	364	"movq %%rbp,(%%rbx)\n\t" \
	365	"addq $-8,(%%rax)\n\t" \
	366	"movq (%%rax), %%rbx\n\t" \
	367	"movq %%rsi,(%%rbx)\n\t" \
	368	"addq $-8,(%%rax)\n\t" \
	369	"movq (%%rax), %%rbx\n\t" \
	370	"movq %%r8,(%%rbx)\n\t" \
	371	"addq $-8,(%%rax)\n\t" \
	372	"movq (%%rax), %%rbx\n\t" \
	373	"movq %%r9,(%%rbx)\n\t" \
	374	"addq $-8,(%%rax)\n\t" \
	375	"movq (%%rax), %%rbx\n\t" \
	376	"movq %%r10,(%%rbx)\n\t" \
	377	"addq $-8,(%%rax)\n\t" \
	378	"movq (%%rax), %%rbx\n\t" \
	379	"movq %%r11,(%%rbx)\n\t" \
	380	"addq $-8,(%%rax)\n\t" \
	381	"movq (%%rax), %%rbx\n\t" \
	382	"movq %%r12,(%%rbx)\n\t" \
	383	"addq $-8,(%%rax)\n\t" \
	384	"movq (%%rax), %%rbx\n\t" \
	385	"movq %%r13,(%%rbx)\n\t" \
	386	"addq $-8,(%%rax)\n\t" \
	387	"movq (%%rax), %%rbx\n\t" \
	388	"movq %%r14,(%%rbx)\n\t" \
	389	"addq $-8,(%%rax)\n\t" \
	390	"movq (%%rax), %%rbx\n\t" \
	391	"movq %%r15,(%%rbx)\n\t" \
636ca5d6 PMF	392	/* push cs */ \
	393	"addq $-2,(%%rax)\n\t" \
	394	"movq (%%rax), %%rbx\n\t" \
	395	"movw %%cs, (%%rbx)\n\t" \
	396	/* push ss */ \
	397	"addq $-2,(%%rax)\n\t" \
	398	"movq (%%rax), %%rbx\n\t" \
	399	"movw %%ss, (%%rbx)\n\t" \
	400	/* add padding for struct registers */ \
	401	"addq $-4,(%%rax)\n\t" \
	402	/* restore original values of regs that were used internally */ \
	403	"popq %%rdi\n\t" \
	404	"popq %%rbx\n\t" \
	405	"popq %%rax\n\t" \
	406	/* cancel push of rsp */ \
	407	"addq $8,%%rsp\n\t" \
a5850bc4 PMF	408	/* cancel push of rflags */ \
a5850bc4 PMF	409	"addq $8,%%rsp\n\t" \
636ca5d6 PMF	410	::); \
	411	memcpy(regsptr, (void *)ust_reg_stack_ptr, sizeof(struct registers)); \
	412	ust_reg_stack_ptr = (void *)(((long)ust_reg_stack_ptr) + sizeof(struct registers));
d98a01c6	413
fc1f31ab PMF	414	#else /* CONFIG_UST_GDB_INTEGRATION */
fc1f31ab PMF	415
fc1f31ab PMF	416	#define save_registers(a)
fc1f31ab PMF	417
e003d6ee	418	#endif /* CONFIG_UST_GDB_INTEGRATION */
defa46a7	419
9e8f4f52 PMF	420	/* Macro to insert the address of a relative jump in an assembly stub,
	421	* in a relocatable way. On x86-64, this uses a special (%rip) notation. */
	422	#define RELATIVE_ADDRESS(__rel_label__) __rel_label__(%%rip)
	423
6fa0d97f	424	#define ARCH_COPY_ADDR(dst) "lea 2b(%%rip)," dst "\n\t"
9692fa18	425
9e8f4f52 PMF	426	#define _ASM_PTR ".quad "
9e8f4f52 PMF	427
6fa0d97f PMF	428	#endif /* x86_64 */
	429
	430	#ifdef __PPC__
	431
	432	struct registers {
	433	};
	434
	435	static __inline__ int fls(unsigned int x)
	436	{
	437	int lz;
	438
	439	asm ("cntlzw %0,%1" : "=r" (lz) : "r" (x));
	440	return 32 - lz;
	441	}
	442
	443	#define ARCH_COPY_ADDR(dst) \
	444	"lis " dst ",2b@h\n\t" /* load high bytes */ \
	445	"ori " dst "," dst ",2b@l\n\t" /* load low bytes */
	446
	447	#define _ASM_PTR ".long "
	448	#define save_registers(a)
	449
	450	#endif /* __PPC__ */
d98a01c6	451
5af57e62	452	#endif /* UST_PROCESSOR_H */