typedef __signed__ char __s8; typedef unsigned char __u8; typedef __signed__ short __s16; typedef unsigned short __u16; typedef __signed__ int __s32; typedef unsigned int __u32; __extension__ typedef __signed__ long long __s64; __extension__ typedef unsigned long long __u64; typedef signed char s8; typedef unsigned char u8; typedef signed short s16; typedef unsigned short u16; typedef signed int s32; typedef unsigned int u32; typedef signed long long s64; typedef unsigned long long u64; typedef unsigned short umode_t; struct ftrace_branch_data { const char *func; const char *file; unsigned line; union { struct { unsigned long correct; unsigned long incorrect; }; struct { unsigned long miss; unsigned long hit; }; unsigned long miss_hit[2]; }; }; void ftrace_likely_update(struct ftrace_branch_data *f, int val, int expect); enum { false = 0, true = 1 }; typedef struct { unsigned long fds_bits [(1024/(8 * sizeof(unsigned long)))]; } __kernel_fd_set; typedef void (*__kernel_sighandler_t)(int); typedef int __kernel_key_t; typedef int __kernel_mqd_t; typedef unsigned long __kernel_ino_t; typedef unsigned short __kernel_mode_t; typedef unsigned short __kernel_nlink_t; typedef long __kernel_off_t; typedef int __kernel_pid_t; typedef unsigned short __kernel_ipc_pid_t; typedef unsigned short __kernel_uid_t; typedef unsigned short __kernel_gid_t; typedef unsigned int __kernel_size_t; typedef int __kernel_ssize_t; typedef int __kernel_ptrdiff_t; typedef long __kernel_time_t; typedef long __kernel_suseconds_t; typedef long __kernel_clock_t; typedef int __kernel_timer_t; typedef int __kernel_clockid_t; typedef int __kernel_daddr_t; typedef char * __kernel_caddr_t; typedef unsigned short __kernel_uid16_t; typedef unsigned short __kernel_gid16_t; typedef unsigned int __kernel_uid32_t; typedef unsigned int __kernel_gid32_t; typedef unsigned short __kernel_old_uid_t; typedef unsigned short __kernel_old_gid_t; typedef unsigned short __kernel_old_dev_t; typedef long long __kernel_loff_t; typedef struct { int val[2]; } __kernel_fsid_t; typedef __u32 __kernel_dev_t; typedef __kernel_fd_set fd_set; typedef __kernel_dev_t dev_t; typedef __kernel_ino_t ino_t; typedef __kernel_mode_t mode_t; typedef __kernel_nlink_t nlink_t; typedef __kernel_off_t off_t; typedef __kernel_pid_t pid_t; typedef __kernel_daddr_t daddr_t; typedef __kernel_key_t key_t; typedef __kernel_suseconds_t suseconds_t; typedef __kernel_timer_t timer_t; typedef __kernel_clockid_t clockid_t; typedef __kernel_mqd_t mqd_t; typedef _Bool bool; typedef __kernel_uid32_t uid_t; typedef __kernel_gid32_t gid_t; typedef __kernel_uid16_t uid16_t; typedef __kernel_gid16_t gid16_t; typedef unsigned long uintptr_t; typedef __kernel_old_uid_t old_uid_t; typedef __kernel_old_gid_t old_gid_t; typedef __kernel_loff_t loff_t; typedef __kernel_size_t size_t; typedef __kernel_ssize_t ssize_t; typedef __kernel_ptrdiff_t ptrdiff_t; typedef __kernel_time_t time_t; typedef __kernel_clock_t clock_t; typedef __kernel_caddr_t caddr_t; typedef unsigned char u_char; typedef unsigned short u_short; typedef unsigned int u_int; typedef unsigned long u_long; typedef unsigned char unchar; typedef unsigned short ushort; typedef unsigned int uint; typedef unsigned long ulong; typedef __u8 u_int8_t; typedef __s8 int8_t; typedef __u16 u_int16_t; typedef __s16 int16_t; typedef __u32 u_int32_t; typedef __s32 int32_t; typedef __u8 uint8_t; typedef __u16 uint16_t; typedef __u32 uint32_t; typedef __u64 uint64_t; typedef __u64 u_int64_t; typedef __s64 int64_t; typedef u64 sector_t; typedef u64 blkcnt_t; typedef u64 dma_addr_t; typedef __u16 __le16; typedef __u16 __be16; typedef __u32 __le32; typedef __u32 __be32; typedef __u64 __le64; typedef __u64 __be64; typedef __u16 __sum16; typedef __u32 __wsum; typedef unsigned gfp_t; typedef unsigned fmode_t; typedef u64 phys_addr_t; typedef phys_addr_t resource_size_t; typedef struct { int counter; } atomic_t; struct list_head { struct list_head *next, *prev; }; struct hlist_head { struct hlist_node *first; }; struct hlist_node { struct hlist_node *next, **pprev; }; struct ustat { __kernel_daddr_t f_tfree; __kernel_ino_t f_tinode; char f_fname[6]; char f_fpack[6]; }; struct timespec; struct compat_timespec; struct restart_block { long (*fn)(struct restart_block *); union { struct { u32 *uaddr; u32 val; u32 flags; u32 bitset; u64 time; u32 *uaddr2; } futex; struct { clockid_t clockid; struct timespec *rmtp; u64 expires; } nanosleep; struct { struct pollfd *ufds; int nfds; int has_timeout; unsigned long tv_sec; unsigned long tv_nsec; } poll; }; }; extern long do_no_restart_syscall(struct restart_block *parm); extern unsigned int __sw_hweight8(unsigned int w); extern unsigned int __sw_hweight16(unsigned int w); extern unsigned int __sw_hweight32(unsigned int w); extern unsigned long __sw_hweight64(__u64 w); struct alt_instr { u8 *instr; u8 *replacement; u16 cpuid; u8 instrlen; u8 replacementlen; }; extern void alternative_instructions(void); extern void apply_alternatives(struct alt_instr *start, struct alt_instr *end); struct module; extern void alternatives_smp_module_add(struct module *mod, char *name, void *locks, void *locks_end, void *text, void *text_end); extern void alternatives_smp_module_del(struct module *mod); extern void alternatives_smp_switch(int smp); extern int alternatives_text_reserved(void *start, void *end); extern bool skip_smp_alternatives; extern const char * const x86_cap_flags[10*32]; extern const char * const x86_power_flags[32]; static inline __attribute__((always_inline)) __attribute__((always_inline)) __attribute__((pure)) bool __static_cpu_has(u16 bit) { asm goto("1: jmp %l[t_no]\n" "2:\n" ".section .altinstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "1b\n" " " ".long" " " "0\n" " .word %P0\n" " .byte 2b - 1b\n" " .byte 0\n" ".previous\n" : : "i" (bit) : : t_no); return true; t_no: return false; } struct paravirt_patch_site; void apply_paravirt(struct paravirt_patch_site *start, struct paravirt_patch_site *end); extern void *text_poke_early(void *addr, const void *opcode, size_t len); struct text_poke_param { void *addr; const void *opcode; size_t len; }; extern void *text_poke(void *addr, const void *opcode, size_t len); extern void *text_poke_smp(void *addr, const void *opcode, size_t len); extern void text_poke_smp_batch(struct text_poke_param *params, int n); static inline __attribute__((always_inline)) __attribute__((always_inline)) void set_bit(unsigned int nr, volatile unsigned long *addr) { if (__builtin_constant_p((((__builtin_constant_p(nr))))) ? !!(((__builtin_constant_p(nr)))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/bitops.h", .line = 62, }; ______r = !!(((__builtin_constant_p(nr)))); ______f.miss_hit[______r]++; ______r; })) { asm volatile(".section .smp_locks,\"a\"\n" ".balign 4\n" ".long 671f - .\n" ".previous\n" "671:" "\n\tlock; " "orb %1,%0" : "+m" (*(volatile long *) ((void *)(addr) + ((nr)>>3))) : "iq" ((u8)(1 << ((nr) & 7))) : "memory"); } else { asm volatile(".section .smp_locks,\"a\"\n" ".balign 4\n" ".long 671f - .\n" ".previous\n" "671:" "\n\tlock; " "bts %1,%0" : "+m" (*(volatile long *) (addr)) : "Ir" (nr) : "memory"); } } static inline __attribute__((always_inline)) void __set_bit(int nr, volatile unsigned long *addr) { asm volatile("bts %1,%0" : "+m" (*(volatile long *) (addr)) : "Ir" (nr) : "memory"); } static inline __attribute__((always_inline)) __attribute__((always_inline)) void clear_bit(int nr, volatile unsigned long *addr) { if (__builtin_constant_p((((__builtin_constant_p(nr))))) ? !!(((__builtin_constant_p(nr)))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/bitops.h", .line = 100, }; ______r = !!(((__builtin_constant_p(nr)))); ______f.miss_hit[______r]++; ______r; })) { asm volatile(".section .smp_locks,\"a\"\n" ".balign 4\n" ".long 671f - .\n" ".previous\n" "671:" "\n\tlock; " "andb %1,%0" : "+m" (*(volatile long *) ((void *)(addr) + ((nr)>>3))) : "iq" ((u8)~(1 << ((nr) & 7)))); } else { asm volatile(".section .smp_locks,\"a\"\n" ".balign 4\n" ".long 671f - .\n" ".previous\n" "671:" "\n\tlock; " "btr %1,%0" : "+m" (*(volatile long *) (addr)) : "Ir" (nr)); } } static inline __attribute__((always_inline)) void clear_bit_unlock(unsigned nr, volatile unsigned long *addr) { __asm__ __volatile__("": : :"memory"); clear_bit(nr, addr); } static inline __attribute__((always_inline)) void __clear_bit(int nr, volatile unsigned long *addr) { asm volatile("btr %1,%0" : "+m" (*(volatile long *) (addr)) : "Ir" (nr)); } static inline __attribute__((always_inline)) void __clear_bit_unlock(unsigned nr, volatile unsigned long *addr) { __asm__ __volatile__("": : :"memory"); __clear_bit(nr, addr); } static inline __attribute__((always_inline)) void __change_bit(int nr, volatile unsigned long *addr) { asm volatile("btc %1,%0" : "+m" (*(volatile long *) (addr)) : "Ir" (nr)); } static inline __attribute__((always_inline)) void change_bit(int nr, volatile unsigned long *addr) { if (__builtin_constant_p((((__builtin_constant_p(nr))))) ? !!(((__builtin_constant_p(nr)))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/bitops.h", .line = 176, }; ______r = !!(((__builtin_constant_p(nr)))); ______f.miss_hit[______r]++; ______r; })) { asm volatile(".section .smp_locks,\"a\"\n" ".balign 4\n" ".long 671f - .\n" ".previous\n" "671:" "\n\tlock; " "xorb %1,%0" : "+m" (*(volatile long *) ((void *)(addr) + ((nr)>>3))) : "iq" ((u8)(1 << ((nr) & 7)))); } else { asm volatile(".section .smp_locks,\"a\"\n" ".balign 4\n" ".long 671f - .\n" ".previous\n" "671:" "\n\tlock; " "btc %1,%0" : "+m" (*(volatile long *) (addr)) : "Ir" (nr)); } } static inline __attribute__((always_inline)) int test_and_set_bit(int nr, volatile unsigned long *addr) { int oldbit; asm volatile(".section .smp_locks,\"a\"\n" ".balign 4\n" ".long 671f - .\n" ".previous\n" "671:" "\n\tlock; " "bts %2,%1\n\t" "sbb %0,%0" : "=r" (oldbit), "+m" (*(volatile long *) (addr)) : "Ir" (nr) : "memory"); return oldbit; } static inline __attribute__((always_inline)) __attribute__((always_inline)) int test_and_set_bit_lock(int nr, volatile unsigned long *addr) { return test_and_set_bit(nr, addr); } static inline __attribute__((always_inline)) int __test_and_set_bit(int nr, volatile unsigned long *addr) { int oldbit; asm("bts %2,%1\n\t" "sbb %0,%0" : "=r" (oldbit), "+m" (*(volatile long *) (addr)) : "Ir" (nr)); return oldbit; } static inline __attribute__((always_inline)) int test_and_clear_bit(int nr, volatile unsigned long *addr) { int oldbit; asm volatile(".section .smp_locks,\"a\"\n" ".balign 4\n" ".long 671f - .\n" ".previous\n" "671:" "\n\tlock; " "btr %2,%1\n\t" "sbb %0,%0" : "=r" (oldbit), "+m" (*(volatile long *) (addr)) : "Ir" (nr) : "memory"); return oldbit; } static inline __attribute__((always_inline)) int __test_and_clear_bit(int nr, volatile unsigned long *addr) { int oldbit; asm volatile("btr %2,%1\n\t" "sbb %0,%0" : "=r" (oldbit), "+m" (*(volatile long *) (addr)) : "Ir" (nr)); return oldbit; } static inline __attribute__((always_inline)) int __test_and_change_bit(int nr, volatile unsigned long *addr) { int oldbit; asm volatile("btc %2,%1\n\t" "sbb %0,%0" : "=r" (oldbit), "+m" (*(volatile long *) (addr)) : "Ir" (nr) : "memory"); return oldbit; } static inline __attribute__((always_inline)) int test_and_change_bit(int nr, volatile unsigned long *addr) { int oldbit; asm volatile(".section .smp_locks,\"a\"\n" ".balign 4\n" ".long 671f - .\n" ".previous\n" "671:" "\n\tlock; " "btc %2,%1\n\t" "sbb %0,%0" : "=r" (oldbit), "+m" (*(volatile long *) (addr)) : "Ir" (nr) : "memory"); return oldbit; } static inline __attribute__((always_inline)) __attribute__((always_inline)) int constant_test_bit(unsigned int nr, const volatile unsigned long *addr) { return ((1UL << (nr % 32)) & (addr[nr / 32])) != 0; } static inline __attribute__((always_inline)) int variable_test_bit(int nr, volatile const unsigned long *addr) { int oldbit; asm volatile("bt %2,%1\n\t" "sbb %0,%0" : "=r" (oldbit) : "m" (*(unsigned long *)addr), "Ir" (nr)); return oldbit; } static inline __attribute__((always_inline)) unsigned long __ffs(unsigned long word) { asm("bsf %1,%0" : "=r" (word) : "rm" (word)); return word; } static inline __attribute__((always_inline)) unsigned long ffz(unsigned long word) { asm("bsf %1,%0" : "=r" (word) : "r" (~word)); return word; } static inline __attribute__((always_inline)) unsigned long __fls(unsigned long word) { asm("bsr %1,%0" : "=r" (word) : "rm" (word)); return word; } static inline __attribute__((always_inline)) int ffs(int x) { int r; asm("bsfl %1,%0\n\t" "cmovzl %2,%0" : "=r" (r) : "rm" (x), "r" (-1)); return r + 1; } static inline __attribute__((always_inline)) int fls(int x) { int r; asm("bsrl %1,%0\n\t" "cmovzl %2,%0" : "=&r" (r) : "rm" (x), "rm" (-1)); return r + 1; } extern unsigned long find_next_bit(const unsigned long *addr, unsigned long size, unsigned long offset); extern unsigned long find_next_zero_bit(const unsigned long *addr, unsigned long size, unsigned long offset); extern unsigned long find_first_bit(const unsigned long *addr, unsigned long size); extern unsigned long find_first_zero_bit(const unsigned long *addr, unsigned long size); static inline __attribute__((always_inline)) int sched_find_first_bit(const unsigned long *b) { if (__builtin_constant_p(((b[0]))) ? !!((b[0])) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/asm-generic/bitops/sched.h", .line = 19, }; ______r = !!((b[0])); ______f.miss_hit[______r]++; ______r; })) return __ffs(b[0]); if (__builtin_constant_p(((b[1]))) ? !!((b[1])) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/asm-generic/bitops/sched.h", .line = 21, }; ______r = !!((b[1])); ______f.miss_hit[______r]++; ______r; })) return __ffs(b[1]) + 32; if (__builtin_constant_p(((b[2]))) ? !!((b[2])) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/asm-generic/bitops/sched.h", .line = 23, }; ______r = !!((b[2])); ______f.miss_hit[______r]++; ______r; })) return __ffs(b[2]) + 64; return __ffs(b[3]) + 96; } static inline __attribute__((always_inline)) unsigned int __arch_hweight32(unsigned int w) { unsigned int res = 0; asm ("661:\n\t" "call __sw_hweight32" "\n662:\n" ".section .altinstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "661b\n" " " ".long" " " "663f\n" " .word " "(4*32+23)" "\n" " .byte 662b-661b\n" " .byte 664f-663f\n" ".previous\n" ".section .discard,\"aw\",@progbits\n" " .byte 0xff + (664f-663f) - (662b-661b)\n" ".previous\n" ".section .altinstr_replacement, \"ax\"\n" "663:\n\t" ".byte 0xf3,0x0f,0xb8,0xc0" "\n664:\n" ".previous" : "=""a" (res) : "a" (w)); return res; } static inline __attribute__((always_inline)) unsigned int __arch_hweight16(unsigned int w) { return __arch_hweight32(w & 0xffff); } static inline __attribute__((always_inline)) unsigned int __arch_hweight8(unsigned int w) { return __arch_hweight32(w & 0xff); } static inline __attribute__((always_inline)) unsigned long __arch_hweight64(__u64 w) { unsigned long res = 0; return __arch_hweight32((u32)w) + __arch_hweight32((u32)(w >> 32)); return res; } static inline __attribute__((always_inline)) __attribute__((always_inline)) int fls64(__u64 x) { __u32 h = x >> 32; if (__builtin_constant_p(((h))) ? !!((h)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/asm-generic/bitops/fls64.h", .line = 21, }; ______r = !!((h)); ______f.miss_hit[______r]++; ______r; })) return fls(h) + 32; return fls(x); } static inline __attribute__((always_inline)) __attribute__((__const__)) __u32 __arch_swab32(__u32 val) { asm("bswap %0" : "=r" (val) : "0" (val)); return val; } static inline __attribute__((always_inline)) __attribute__((__const__)) __u64 __arch_swab64(__u64 val) { union { struct { __u32 a; __u32 b; } s; __u64 u; } v; v.u = val; asm("bswapl %0 ; bswapl %1 ; xchgl %0,%1" : "=r" (v.s.a), "=r" (v.s.b) : "0" (v.s.a), "1" (v.s.b)); return v.u; } static inline __attribute__((always_inline)) __attribute__((__const__)) __u16 __fswab16(__u16 val) { return ((__u16)( (((__u16)(val) & (__u16)0x00ffU) << 8) | (((__u16)(val) & (__u16)0xff00U) >> 8))); } static inline __attribute__((always_inline)) __attribute__((__const__)) __u32 __fswab32(__u32 val) { return __arch_swab32(val); } static inline __attribute__((always_inline)) __attribute__((__const__)) __u64 __fswab64(__u64 val) { return __arch_swab64(val); } static inline __attribute__((always_inline)) __attribute__((__const__)) __u32 __fswahw32(__u32 val) { return ((__u32)( (((__u32)(val) & (__u32)0x0000ffffUL) << 16) | (((__u32)(val) & (__u32)0xffff0000UL) >> 16))); } static inline __attribute__((always_inline)) __attribute__((__const__)) __u32 __fswahb32(__u32 val) { return ((__u32)( (((__u32)(val) & (__u32)0x00ff00ffUL) << 8) | (((__u32)(val) & (__u32)0xff00ff00UL) >> 8))); } static inline __attribute__((always_inline)) __u16 __swab16p(const __u16 *p) { return (__builtin_constant_p((__u16)(*p)) ? ((__u16)( (((__u16)(*p) & (__u16)0x00ffU) << 8) | (((__u16)(*p) & (__u16)0xff00U) >> 8))) : __fswab16(*p)); } static inline __attribute__((always_inline)) __u32 __swab32p(const __u32 *p) { return (__builtin_constant_p((__u32)(*p)) ? ((__u32)( (((__u32)(*p) & (__u32)0x000000ffUL) << 24) | (((__u32)(*p) & (__u32)0x0000ff00UL) << 8) | (((__u32)(*p) & (__u32)0x00ff0000UL) >> 8) | (((__u32)(*p) & (__u32)0xff000000UL) >> 24))) : __fswab32(*p)); } static inline __attribute__((always_inline)) __u64 __swab64p(const __u64 *p) { return (__builtin_constant_p((__u64)(*p)) ? ((__u64)( (((__u64)(*p) & (__u64)0x00000000000000ffULL) << 56) | (((__u64)(*p) & (__u64)0x000000000000ff00ULL) << 40) | (((__u64)(*p) & (__u64)0x0000000000ff0000ULL) << 24) | (((__u64)(*p) & (__u64)0x00000000ff000000ULL) << 8) | (((__u64)(*p) & (__u64)0x000000ff00000000ULL) >> 8) | (((__u64)(*p) & (__u64)0x0000ff0000000000ULL) >> 24) | (((__u64)(*p) & (__u64)0x00ff000000000000ULL) >> 40) | (((__u64)(*p) & (__u64)0xff00000000000000ULL) >> 56))) : __fswab64(*p)); } static inline __attribute__((always_inline)) __u32 __swahw32p(const __u32 *p) { return (__builtin_constant_p((__u32)(*p)) ? ((__u32)( (((__u32)(*p) & (__u32)0x0000ffffUL) << 16) | (((__u32)(*p) & (__u32)0xffff0000UL) >> 16))) : __fswahw32(*p)); } static inline __attribute__((always_inline)) __u32 __swahb32p(const __u32 *p) { return (__builtin_constant_p((__u32)(*p)) ? ((__u32)( (((__u32)(*p) & (__u32)0x00ff00ffUL) << 8) | (((__u32)(*p) & (__u32)0xff00ff00UL) >> 8))) : __fswahb32(*p)); } static inline __attribute__((always_inline)) void __swab16s(__u16 *p) { *p = __swab16p(p); } static inline __attribute__((always_inline)) void __swab32s(__u32 *p) { *p = __swab32p(p); } static inline __attribute__((always_inline)) void __swab64s(__u64 *p) { *p = __swab64p(p); } static inline __attribute__((always_inline)) void __swahw32s(__u32 *p) { *p = __swahw32p(p); } static inline __attribute__((always_inline)) void __swahb32s(__u32 *p) { *p = __swahb32p(p); } static inline __attribute__((always_inline)) __le64 __cpu_to_le64p(const __u64 *p) { return ( __le64)*p; } static inline __attribute__((always_inline)) __u64 __le64_to_cpup(const __le64 *p) { return ( __u64)*p; } static inline __attribute__((always_inline)) __le32 __cpu_to_le32p(const __u32 *p) { return ( __le32)*p; } static inline __attribute__((always_inline)) __u32 __le32_to_cpup(const __le32 *p) { return ( __u32)*p; } static inline __attribute__((always_inline)) __le16 __cpu_to_le16p(const __u16 *p) { return ( __le16)*p; } static inline __attribute__((always_inline)) __u16 __le16_to_cpup(const __le16 *p) { return ( __u16)*p; } static inline __attribute__((always_inline)) __be64 __cpu_to_be64p(const __u64 *p) { return ( __be64)__swab64p(p); } static inline __attribute__((always_inline)) __u64 __be64_to_cpup(const __be64 *p) { return __swab64p((__u64 *)p); } static inline __attribute__((always_inline)) __be32 __cpu_to_be32p(const __u32 *p) { return ( __be32)__swab32p(p); } static inline __attribute__((always_inline)) __u32 __be32_to_cpup(const __be32 *p) { return __swab32p((__u32 *)p); } static inline __attribute__((always_inline)) __be16 __cpu_to_be16p(const __u16 *p) { return ( __be16)__swab16p(p); } static inline __attribute__((always_inline)) __u16 __be16_to_cpup(const __be16 *p) { return __swab16p((__u16 *)p); } static inline __attribute__((always_inline)) void le16_add_cpu(__le16 *var, u16 val) { *var = (( __le16)(__u16)((( __u16)(__le16)(*var)) + val)); } static inline __attribute__((always_inline)) void le32_add_cpu(__le32 *var, u32 val) { *var = (( __le32)(__u32)((( __u32)(__le32)(*var)) + val)); } static inline __attribute__((always_inline)) void le64_add_cpu(__le64 *var, u64 val) { *var = (( __le64)(__u64)((( __u64)(__le64)(*var)) + val)); } static inline __attribute__((always_inline)) void be16_add_cpu(__be16 *var, u16 val) { *var = (( __be16)(__builtin_constant_p((__u16)(((__builtin_constant_p((__u16)(( __u16)(__be16)(*var))) ? ((__u16)( (((__u16)(( __u16)(__be16)(*var)) & (__u16)0x00ffU) << 8) | (((__u16)(( __u16)(__be16)(*var)) & (__u16)0xff00U) >> 8))) : __fswab16(( __u16)(__be16)(*var))) + val))) ? ((__u16)( (((__u16)(((__builtin_constant_p((__u16)(( __u16)(__be16)(*var))) ? ((__u16)( (((__u16)(( __u16)(__be16)(*var)) & (__u16)0x00ffU) << 8) | (((__u16)(( __u16)(__be16)(*var)) & (__u16)0xff00U) >> 8))) : __fswab16(( __u16)(__be16)(*var))) + val)) & (__u16)0x00ffU) << 8) | (((__u16)(((__builtin_constant_p((__u16)(( __u16)(__be16)(*var))) ? ((__u16)( (((__u16)(( __u16)(__be16)(*var)) & (__u16)0x00ffU) << 8) | (((__u16)(( __u16)(__be16)(*var)) & (__u16)0xff00U) >> 8))) : __fswab16(( __u16)(__be16)(*var))) + val)) & (__u16)0xff00U) >> 8))) : __fswab16(((__builtin_constant_p((__u16)(( __u16)(__be16)(*var))) ? ((__u16)( (((__u16)(( __u16)(__be16)(*var)) & (__u16)0x00ffU) << 8) | (((__u16)(( __u16)(__be16)(*var)) & (__u16)0xff00U) >> 8))) : __fswab16(( __u16)(__be16)(*var))) + val)))); } static inline __attribute__((always_inline)) void be32_add_cpu(__be32 *var, u32 val) { *var = (( __be32)(__builtin_constant_p((__u32)(((__builtin_constant_p((__u32)(( __u32)(__be32)(*var))) ? ((__u32)( (((__u32)(( __u32)(__be32)(*var)) & (__u32)0x000000ffUL) << 24) | (((__u32)(( __u32)(__be32)(*var)) & (__u32)0x0000ff00UL) << 8) | (((__u32)(( __u32)(__be32)(*var)) & (__u32)0x00ff0000UL) >> 8) | (((__u32)(( __u32)(__be32)(*var)) & (__u32)0xff000000UL) >> 24))) : __fswab32(( __u32)(__be32)(*var))) + val))) ? ((__u32)( (((__u32)(((__builtin_constant_p((__u32)(( __u32)(__be32)(*var))) ? ((__u32)( (((__u32)(( __u32)(__be32)(*var)) & (__u32)0x000000ffUL) << 24) | (((__u32)(( __u32)(__be32)(*var)) & (__u32)0x0000ff00UL) << 8) | (((__u32)(( __u32)(__be32)(*var)) & (__u32)0x00ff0000UL) >> 8) | (((__u32)(( __u32)(__be32)(*var)) & (__u32)0xff000000UL) >> 24))) : __fswab32(( __u32)(__be32)(*var))) + val)) & (__u32)0x000000ffUL) << 24) | (((__u32)(((__builtin_constant_p((__u32)(( __u32)(__be32)(*var))) ? ((__u32)( (((__u32)(( __u32)(__be32)(*var)) & (__u32)0x000000ffUL) << 24) | (((__u32)(( __u32)(__be32)(*var)) & (__u32)0x0000ff00UL) << 8) | (((__u32)(( __u32)(__be32)(*var)) & (__u32)0x00ff0000UL) >> 8) | (((__u32)(( __u32)(__be32)(*var)) & (__u32)0xff000000UL) >> 24))) : __fswab32(( __u32)(__be32)(*var))) + val)) & (__u32)0x0000ff00UL) << 8) | (((__u32)(((__builtin_constant_p((__u32)(( __u32)(__be32)(*var))) ? ((__u32)( (((__u32)(( __u32)(__be32)(*var)) & (__u32)0x000000ffUL) << 24) | (((__u32)(( __u32)(__be32)(*var)) & (__u32)0x0000ff00UL) << 8) | (((__u32)(( __u32)(__be32)(*var)) & (__u32)0x00ff0000UL) >> 8) | (((__u32)(( __u32)(__be32)(*var)) & (__u32)0xff000000UL) >> 24))) : __fswab32(( __u32)(__be32)(*var))) + val)) & (__u32)0x00ff0000UL) >> 8) | (((__u32)(((__builtin_constant_p((__u32)(( __u32)(__be32)(*var))) ? ((__u32)( (((__u32)(( __u32)(__be32)(*var)) & (__u32)0x000000ffUL) << 24) | (((__u32)(( __u32)(__be32)(*var)) & (__u32)0x0000ff00UL) << 8) | (((__u32)(( __u32)(__be32)(*var)) & (__u32)0x00ff0000UL) >> 8) | (((__u32)(( __u32)(__be32)(*var)) & (__u32)0xff000000UL) >> 24))) : __fswab32(( __u32)(__be32)(*var))) + val)) & (__u32)0xff000000UL) >> 24))) : __fswab32(((__builtin_constant_p((__u32)(( __u32)(__be32)(*var))) ? ((__u32)( (((__u32)(( __u32)(__be32)(*var)) & (__u32)0x000000ffUL) << 24) | (((__u32)(( __u32)(__be32)(*var)) & (__u32)0x0000ff00UL) << 8) | (((__u32)(( __u32)(__be32)(*var)) & (__u32)0x00ff0000UL) >> 8) | (((__u32)(( __u32)(__be32)(*var)) & (__u32)0xff000000UL) >> 24))) : __fswab32(( __u32)(__be32)(*var))) + val)))); } static inline __attribute__((always_inline)) void be64_add_cpu(__be64 *var, u64 val) { *var = (( __be64)(__builtin_constant_p((__u64)(((__builtin_constant_p((__u64)(( __u64)(__be64)(*var))) ? ((__u64)( (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x00000000000000ffULL) << 56) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x000000000000ff00ULL) << 40) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x0000000000ff0000ULL) << 24) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x00000000ff000000ULL) << 8) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x000000ff00000000ULL) >> 8) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x0000ff0000000000ULL) >> 24) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x00ff000000000000ULL) >> 40) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0xff00000000000000ULL) >> 56))) : __fswab64(( __u64)(__be64)(*var))) + val))) ? ((__u64)( (((__u64)(((__builtin_constant_p((__u64)(( __u64)(__be64)(*var))) ? ((__u64)( (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x00000000000000ffULL) << 56) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x000000000000ff00ULL) << 40) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x0000000000ff0000ULL) << 24) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x00000000ff000000ULL) << 8) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x000000ff00000000ULL) >> 8) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x0000ff0000000000ULL) >> 24) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x00ff000000000000ULL) >> 40) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0xff00000000000000ULL) >> 56))) : __fswab64(( __u64)(__be64)(*var))) + val)) & (__u64)0x00000000000000ffULL) << 56) | (((__u64)(((__builtin_constant_p((__u64)(( __u64)(__be64)(*var))) ? ((__u64)( (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x00000000000000ffULL) << 56) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x000000000000ff00ULL) << 40) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x0000000000ff0000ULL) << 24) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x00000000ff000000ULL) << 8) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x000000ff00000000ULL) >> 8) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x0000ff0000000000ULL) >> 24) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x00ff000000000000ULL) >> 40) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0xff00000000000000ULL) >> 56))) : __fswab64(( __u64)(__be64)(*var))) + val)) & (__u64)0x000000000000ff00ULL) << 40) | (((__u64)(((__builtin_constant_p((__u64)(( __u64)(__be64)(*var))) ? ((__u64)( (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x00000000000000ffULL) << 56) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x000000000000ff00ULL) << 40) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x0000000000ff0000ULL) << 24) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x00000000ff000000ULL) << 8) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x000000ff00000000ULL) >> 8) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x0000ff0000000000ULL) >> 24) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x00ff000000000000ULL) >> 40) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0xff00000000000000ULL) >> 56))) : __fswab64(( __u64)(__be64)(*var))) + val)) & (__u64)0x0000000000ff0000ULL) << 24) | (((__u64)(((__builtin_constant_p((__u64)(( __u64)(__be64)(*var))) ? ((__u64)( (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x00000000000000ffULL) << 56) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x000000000000ff00ULL) << 40) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x0000000000ff0000ULL) << 24) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x00000000ff000000ULL) << 8) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x000000ff00000000ULL) >> 8) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x0000ff0000000000ULL) >> 24) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x00ff000000000000ULL) >> 40) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0xff00000000000000ULL) >> 56))) : __fswab64(( __u64)(__be64)(*var))) + val)) & (__u64)0x00000000ff000000ULL) << 8) | (((__u64)(((__builtin_constant_p((__u64)(( __u64)(__be64)(*var))) ? ((__u64)( (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x00000000000000ffULL) << 56) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x000000000000ff00ULL) << 40) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x0000000000ff0000ULL) << 24) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x00000000ff000000ULL) << 8) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x000000ff00000000ULL) >> 8) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x0000ff0000000000ULL) >> 24) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x00ff000000000000ULL) >> 40) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0xff00000000000000ULL) >> 56))) : __fswab64(( __u64)(__be64)(*var))) + val)) & (__u64)0x000000ff00000000ULL) >> 8) | (((__u64)(((__builtin_constant_p((__u64)(( __u64)(__be64)(*var))) ? ((__u64)( (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x00000000000000ffULL) << 56) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x000000000000ff00ULL) << 40) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x0000000000ff0000ULL) << 24) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x00000000ff000000ULL) << 8) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x000000ff00000000ULL) >> 8) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x0000ff0000000000ULL) >> 24) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x00ff000000000000ULL) >> 40) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0xff00000000000000ULL) >> 56))) : __fswab64(( __u64)(__be64)(*var))) + val)) & (__u64)0x0000ff0000000000ULL) >> 24) | (((__u64)(((__builtin_constant_p((__u64)(( __u64)(__be64)(*var))) ? ((__u64)( (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x00000000000000ffULL) << 56) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x000000000000ff00ULL) << 40) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x0000000000ff0000ULL) << 24) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x00000000ff000000ULL) << 8) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x000000ff00000000ULL) >> 8) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x0000ff0000000000ULL) >> 24) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x00ff000000000000ULL) >> 40) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0xff00000000000000ULL) >> 56))) : __fswab64(( __u64)(__be64)(*var))) + val)) & (__u64)0x00ff000000000000ULL) >> 40) | (((__u64)(((__builtin_constant_p((__u64)(( __u64)(__be64)(*var))) ? ((__u64)( (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x00000000000000ffULL) << 56) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x000000000000ff00ULL) << 40) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x0000000000ff0000ULL) << 24) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x00000000ff000000ULL) << 8) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x000000ff00000000ULL) >> 8) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x0000ff0000000000ULL) >> 24) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x00ff000000000000ULL) >> 40) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0xff00000000000000ULL) >> 56))) : __fswab64(( __u64)(__be64)(*var))) + val)) & (__u64)0xff00000000000000ULL) >> 56))) : __fswab64(((__builtin_constant_p((__u64)(( __u64)(__be64)(*var))) ? ((__u64)( (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x00000000000000ffULL) << 56) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x000000000000ff00ULL) << 40) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x0000000000ff0000ULL) << 24) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x00000000ff000000ULL) << 8) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x000000ff00000000ULL) >> 8) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x0000ff0000000000ULL) >> 24) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0x00ff000000000000ULL) >> 40) | (((__u64)(( __u64)(__be64)(*var)) & (__u64)0xff00000000000000ULL) >> 56))) : __fswab64(( __u64)(__be64)(*var))) + val)))); } static inline __attribute__((always_inline)) unsigned long find_next_zero_bit_le(const void *addr, unsigned long size, unsigned long offset) { return find_next_zero_bit(addr, size, offset); } static inline __attribute__((always_inline)) unsigned long find_next_bit_le(const void *addr, unsigned long size, unsigned long offset) { return find_next_bit(addr, size, offset); } static inline __attribute__((always_inline)) unsigned long find_first_zero_bit_le(const void *addr, unsigned long size) { return find_first_zero_bit(addr, size); } static inline __attribute__((always_inline)) int test_bit_le(int nr, const void *addr) { return (__builtin_constant_p((nr ^ 0)) ? constant_test_bit((nr ^ 0), (addr)) : variable_test_bit((nr ^ 0), (addr))); } static inline __attribute__((always_inline)) void __set_bit_le(int nr, void *addr) { __set_bit(nr ^ 0, addr); } static inline __attribute__((always_inline)) void __clear_bit_le(int nr, void *addr) { __clear_bit(nr ^ 0, addr); } static inline __attribute__((always_inline)) int test_and_set_bit_le(int nr, void *addr) { return test_and_set_bit(nr ^ 0, addr); } static inline __attribute__((always_inline)) int test_and_clear_bit_le(int nr, void *addr) { return test_and_clear_bit(nr ^ 0, addr); } static inline __attribute__((always_inline)) int __test_and_set_bit_le(int nr, void *addr) { return __test_and_set_bit(nr ^ 0, addr); } static inline __attribute__((always_inline)) int __test_and_clear_bit_le(int nr, void *addr) { return __test_and_clear_bit(nr ^ 0, addr); } static __inline__ __attribute__((always_inline)) int get_bitmask_order(unsigned int count) { int order; order = fls(count); return order; } static __inline__ __attribute__((always_inline)) int get_count_order(unsigned int count) { int order; order = fls(count) - 1; if (__builtin_constant_p(((count & (count - 1)))) ? !!((count & (count - 1))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/bitops.h", .line = 42, }; ______r = !!((count & (count - 1))); ______f.miss_hit[______r]++; ______r; })) order++; return order; } static inline __attribute__((always_inline)) unsigned long hweight_long(unsigned long w) { return sizeof(w) == 4 ? (__builtin_constant_p(w) ? ((( (!!((w) & (1ULL << 0))) + (!!((w) & (1ULL << 1))) + (!!((w) & (1ULL << 2))) + (!!((w) & (1ULL << 3))) + (!!((w) & (1ULL << 4))) + (!!((w) & (1ULL << 5))) + (!!((w) & (1ULL << 6))) + (!!((w) & (1ULL << 7))) ) + ( (!!(((w) >> 8) & (1ULL << 0))) + (!!(((w) >> 8) & (1ULL << 1))) + (!!(((w) >> 8) & (1ULL << 2))) + (!!(((w) >> 8) & (1ULL << 3))) + (!!(((w) >> 8) & (1ULL << 4))) + (!!(((w) >> 8) & (1ULL << 5))) + (!!(((w) >> 8) & (1ULL << 6))) + (!!(((w) >> 8) & (1ULL << 7))) )) + (( (!!(((w) >> 16) & (1ULL << 0))) + (!!(((w) >> 16) & (1ULL << 1))) + (!!(((w) >> 16) & (1ULL << 2))) + (!!(((w) >> 16) & (1ULL << 3))) + (!!(((w) >> 16) & (1ULL << 4))) + (!!(((w) >> 16) & (1ULL << 5))) + (!!(((w) >> 16) & (1ULL << 6))) + (!!(((w) >> 16) & (1ULL << 7))) ) + ( (!!((((w) >> 16) >> 8) & (1ULL << 0))) + (!!((((w) >> 16) >> 8) & (1ULL << 1))) + (!!((((w) >> 16) >> 8) & (1ULL << 2))) + (!!((((w) >> 16) >> 8) & (1ULL << 3))) + (!!((((w) >> 16) >> 8) & (1ULL << 4))) + (!!((((w) >> 16) >> 8) & (1ULL << 5))) + (!!((((w) >> 16) >> 8) & (1ULL << 6))) + (!!((((w) >> 16) >> 8) & (1ULL << 7))) ))) : __arch_hweight32(w)) : (__builtin_constant_p(w) ? (((( (!!((w) & (1ULL << 0))) + (!!((w) & (1ULL << 1))) + (!!((w) & (1ULL << 2))) + (!!((w) & (1ULL << 3))) + (!!((w) & (1ULL << 4))) + (!!((w) & (1ULL << 5))) + (!!((w) & (1ULL << 6))) + (!!((w) & (1ULL << 7))) ) + ( (!!(((w) >> 8) & (1ULL << 0))) + (!!(((w) >> 8) & (1ULL << 1))) + (!!(((w) >> 8) & (1ULL << 2))) + (!!(((w) >> 8) & (1ULL << 3))) + (!!(((w) >> 8) & (1ULL << 4))) + (!!(((w) >> 8) & (1ULL << 5))) + (!!(((w) >> 8) & (1ULL << 6))) + (!!(((w) >> 8) & (1ULL << 7))) )) + (( (!!(((w) >> 16) & (1ULL << 0))) + (!!(((w) >> 16) & (1ULL << 1))) + (!!(((w) >> 16) & (1ULL << 2))) + (!!(((w) >> 16) & (1ULL << 3))) + (!!(((w) >> 16) & (1ULL << 4))) + (!!(((w) >> 16) & (1ULL << 5))) + (!!(((w) >> 16) & (1ULL << 6))) + (!!(((w) >> 16) & (1ULL << 7))) ) + ( (!!((((w) >> 16) >> 8) & (1ULL << 0))) + (!!((((w) >> 16) >> 8) & (1ULL << 1))) + (!!((((w) >> 16) >> 8) & (1ULL << 2))) + (!!((((w) >> 16) >> 8) & (1ULL << 3))) + (!!((((w) >> 16) >> 8) & (1ULL << 4))) + (!!((((w) >> 16) >> 8) & (1ULL << 5))) + (!!((((w) >> 16) >> 8) & (1ULL << 6))) + (!!((((w) >> 16) >> 8) & (1ULL << 7))) ))) + ((( (!!(((w) >> 32) & (1ULL << 0))) + (!!(((w) >> 32) & (1ULL << 1))) + (!!(((w) >> 32) & (1ULL << 2))) + (!!(((w) >> 32) & (1ULL << 3))) + (!!(((w) >> 32) & (1ULL << 4))) + (!!(((w) >> 32) & (1ULL << 5))) + (!!(((w) >> 32) & (1ULL << 6))) + (!!(((w) >> 32) & (1ULL << 7))) ) + ( (!!((((w) >> 32) >> 8) & (1ULL << 0))) + (!!((((w) >> 32) >> 8) & (1ULL << 1))) + (!!((((w) >> 32) >> 8) & (1ULL << 2))) + (!!((((w) >> 32) >> 8) & (1ULL << 3))) + (!!((((w) >> 32) >> 8) & (1ULL << 4))) + (!!((((w) >> 32) >> 8) & (1ULL << 5))) + (!!((((w) >> 32) >> 8) & (1ULL << 6))) + (!!((((w) >> 32) >> 8) & (1ULL << 7))) )) + (( (!!((((w) >> 32) >> 16) & (1ULL << 0))) + (!!((((w) >> 32) >> 16) & (1ULL << 1))) + (!!((((w) >> 32) >> 16) & (1ULL << 2))) + (!!((((w) >> 32) >> 16) & (1ULL << 3))) + (!!((((w) >> 32) >> 16) & (1ULL << 4))) + (!!((((w) >> 32) >> 16) & (1ULL << 5))) + (!!((((w) >> 32) >> 16) & (1ULL << 6))) + (!!((((w) >> 32) >> 16) & (1ULL << 7))) ) + ( (!!(((((w) >> 32) >> 16) >> 8) & (1ULL << 0))) + (!!(((((w) >> 32) >> 16) >> 8) & (1ULL << 1))) + (!!(((((w) >> 32) >> 16) >> 8) & (1ULL << 2))) + (!!(((((w) >> 32) >> 16) >> 8) & (1ULL << 3))) + (!!(((((w) >> 32) >> 16) >> 8) & (1ULL << 4))) + (!!(((((w) >> 32) >> 16) >> 8) & (1ULL << 5))) + (!!(((((w) >> 32) >> 16) >> 8) & (1ULL << 6))) + (!!(((((w) >> 32) >> 16) >> 8) & (1ULL << 7))) )))) : __arch_hweight64(w)); } static inline __attribute__((always_inline)) __u32 rol32(__u32 word, unsigned int shift) { return (word << shift) | (word >> (32 - shift)); } static inline __attribute__((always_inline)) __u32 ror32(__u32 word, unsigned int shift) { return (word >> shift) | (word << (32 - shift)); } static inline __attribute__((always_inline)) __u16 rol16(__u16 word, unsigned int shift) { return (word << shift) | (word >> (16 - shift)); } static inline __attribute__((always_inline)) __u16 ror16(__u16 word, unsigned int shift) { return (word >> shift) | (word << (16 - shift)); } static inline __attribute__((always_inline)) __u8 rol8(__u8 word, unsigned int shift) { return (word << shift) | (word >> (8 - shift)); } static inline __attribute__((always_inline)) __u8 ror8(__u8 word, unsigned int shift) { return (word >> shift) | (word << (8 - shift)); } static inline __attribute__((always_inline)) __s32 sign_extend32(__u32 value, int index) { __u8 shift = 31 - index; return (__s32)(value << shift) >> shift; } static inline __attribute__((always_inline)) unsigned fls_long(unsigned long l) { if (__builtin_constant_p(((sizeof(l) == 4))) ? !!((sizeof(l) == 4)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/bitops.h", .line = 125, }; ______r = !!((sizeof(l) == 4)); ______f.miss_hit[______r]++; ______r; })) return fls(l); return fls64(l); } static inline __attribute__((always_inline)) unsigned long __ffs64(u64 word) { if (__builtin_constant_p(((((u32)word) == 0UL))) ? !!((((u32)word) == 0UL)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/bitops.h", .line = 141, }; ______r = !!((((u32)word) == 0UL)); ______f.miss_hit[______r]++; ______r; })) return __ffs((u32)(word >> 32)) + 32; return __ffs((unsigned long)word); } extern unsigned long find_last_bit(const unsigned long *addr, unsigned long size); extern unsigned int __VMALLOC_RESERVE; extern int sysctl_legacy_va_layout; extern void find_low_pfn_range(void); extern void setup_bootmem_allocator(void); extern int devmem_is_allowed(unsigned long pagenr); extern unsigned long max_low_pfn_mapped; extern unsigned long max_pfn_mapped; static inline __attribute__((always_inline)) phys_addr_t get_max_mapped(void) { return (phys_addr_t)max_pfn_mapped << 12; } extern unsigned long init_memory_mapping(unsigned long start, unsigned long end); extern void initmem_init(void); extern void free_initmem(void); typedef __builtin_va_list __gnuc_va_list; typedef __gnuc_va_list va_list; extern char *strndup_user(const char *, long); extern void *memdup_user(const void *, size_t); extern char *strcpy(char *dest, const char *src); extern char *strncpy(char *dest, const char *src, size_t count); extern char *strcat(char *dest, const char *src); extern char *strncat(char *dest, const char *src, size_t count); extern int strcmp(const char *cs, const char *ct); extern int strncmp(const char *cs, const char *ct, size_t count); extern char *strchr(const char *s, int c); extern size_t strlen(const char *s); static inline __attribute__((always_inline)) __attribute__((always_inline)) void *__memcpy(void *to, const void *from, size_t n) { int d0, d1, d2; asm volatile("rep ; movsl\n\t" "movl %4,%%ecx\n\t" "andl $3,%%ecx\n\t" "jz 1f\n\t" "rep ; movsb\n\t" "1:" : "=&c" (d0), "=&D" (d1), "=&S" (d2) : "0" (n / 4), "g" (n), "1" ((long)to), "2" ((long)from) : "memory"); return to; } static inline __attribute__((always_inline)) __attribute__((always_inline)) void *__constant_memcpy(void *to, const void *from, size_t n) { long esi, edi; if (__builtin_constant_p(((!n))) ? !!((!n)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/string_32.h", .line = 55, }; ______r = !!((!n)); ______f.miss_hit[______r]++; ______r; })) return to; switch (n) { case 1: *(char *)to = *(char *)from; return to; case 2: *(short *)to = *(short *)from; return to; case 4: *(int *)to = *(int *)from; return to; case 3: *(short *)to = *(short *)from; *((char *)to + 2) = *((char *)from + 2); return to; case 5: *(int *)to = *(int *)from; *((char *)to + 4) = *((char *)from + 4); return to; case 6: *(int *)to = *(int *)from; *((short *)to + 2) = *((short *)from + 2); return to; case 8: *(int *)to = *(int *)from; *((int *)to + 1) = *((int *)from + 1); return to; } esi = (long)from; edi = (long)to; if (__builtin_constant_p(((n >= 5 * 4))) ? !!((n >= 5 * 4)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/string_32.h", .line = 88, }; ______r = !!((n >= 5 * 4)); ______f.miss_hit[______r]++; ______r; })) { int ecx; asm volatile("rep ; movsl" : "=&c" (ecx), "=&D" (edi), "=&S" (esi) : "0" (n / 4), "1" (edi), "2" (esi) : "memory" ); } else { if (__builtin_constant_p(((n >= 4 * 4))) ? !!((n >= 4 * 4)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/string_32.h", .line = 98, }; ______r = !!((n >= 4 * 4)); ______f.miss_hit[______r]++; ______r; })) asm volatile("movsl" : "=&D"(edi), "=&S"(esi) : "0"(edi), "1"(esi) : "memory"); if (__builtin_constant_p(((n >= 3 * 4))) ? !!((n >= 3 * 4)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/string_32.h", .line = 103, }; ______r = !!((n >= 3 * 4)); ______f.miss_hit[______r]++; ______r; })) asm volatile("movsl" : "=&D"(edi), "=&S"(esi) : "0"(edi), "1"(esi) : "memory"); if (__builtin_constant_p(((n >= 2 * 4))) ? !!((n >= 2 * 4)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/string_32.h", .line = 108, }; ______r = !!((n >= 2 * 4)); ______f.miss_hit[______r]++; ______r; })) asm volatile("movsl" : "=&D"(edi), "=&S"(esi) : "0"(edi), "1"(esi) : "memory"); if (__builtin_constant_p(((n >= 1 * 4))) ? !!((n >= 1 * 4)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/string_32.h", .line = 113, }; ______r = !!((n >= 1 * 4)); ______f.miss_hit[______r]++; ______r; })) asm volatile("movsl" : "=&D"(edi), "=&S"(esi) : "0"(edi), "1"(esi) : "memory"); } switch (n % 4) { case 0: return to; case 1: asm volatile("movsb" : "=&D"(edi), "=&S"(esi) : "0"(edi), "1"(esi) : "memory"); return to; case 2: asm volatile("movsw" : "=&D"(edi), "=&S"(esi) : "0"(edi), "1"(esi) : "memory"); return to; default: asm volatile("movsw\n\tmovsb" : "=&D"(edi), "=&S"(esi) : "0"(edi), "1"(esi) : "memory"); return to; } } void *memmove(void *dest, const void *src, size_t n); extern void *memchr(const void *cs, int c, size_t count); static inline __attribute__((always_inline)) void *__memset_generic(void *s, char c, size_t count) { int d0, d1; asm volatile("rep\n\t" "stosb" : "=&c" (d0), "=&D" (d1) : "a" (c), "1" (s), "0" (count) : "memory"); return s; } static inline __attribute__((always_inline)) __attribute__((always_inline)) void *__constant_c_memset(void *s, unsigned long c, size_t count) { int d0, d1; asm volatile("rep ; stosl\n\t" "testb $2,%b3\n\t" "je 1f\n\t" "stosw\n" "1:\ttestb $1,%b3\n\t" "je 2f\n\t" "stosb\n" "2:" : "=&c" (d0), "=&D" (d1) : "a" (c), "q" (count), "0" (count/4), "1" ((long)s) : "memory"); return s; } extern size_t strnlen(const char *s, size_t count); extern char *strstr(const char *cs, const char *ct); static inline __attribute__((always_inline)) __attribute__((always_inline)) void *__constant_c_and_count_memset(void *s, unsigned long pattern, size_t count) { switch (count) { case 0: return s; case 1: *(unsigned char *)s = pattern & 0xff; return s; case 2: *(unsigned short *)s = pattern & 0xffff; return s; case 3: *(unsigned short *)s = pattern & 0xffff; *((unsigned char *)s + 2) = pattern & 0xff; return s; case 4: *(unsigned long *)s = pattern; return s; } { int d0, d1; unsigned long eax = pattern; switch (count % 4) { case 0: asm volatile("rep ; stosl" "" : "=&c" (d0), "=&D" (d1) : "a" (eax), "0" (count/4), "1" ((long)s) : "memory"); return s; case 1: asm volatile("rep ; stosl" "\n\tstosb" : "=&c" (d0), "=&D" (d1) : "a" (eax), "0" (count/4), "1" ((long)s) : "memory"); return s; case 2: asm volatile("rep ; stosl" "\n\tstosw" : "=&c" (d0), "=&D" (d1) : "a" (eax), "0" (count/4), "1" ((long)s) : "memory"); return s; default: asm volatile("rep ; stosl" "\n\tstosw\n\tstosb" : "=&c" (d0), "=&D" (d1) : "a" (eax), "0" (count/4), "1" ((long)s) : "memory"); return s; } } } extern void *memscan(void *addr, int c, size_t size); size_t strlcpy(char *, const char *, size_t); extern size_t strlcat(char *, const char *, __kernel_size_t); extern int strnicmp(const char *, const char *, __kernel_size_t); extern int strcasecmp(const char *s1, const char *s2); extern int strncasecmp(const char *s1, const char *s2, size_t n); extern char * strnchr(const char *, size_t, int); extern char * strrchr(const char *,int); extern char * __attribute__((warn_unused_result)) skip_spaces(const char *); extern char *strim(char *); static inline __attribute__((always_inline)) __attribute__((warn_unused_result)) char *strstrip(char *str) { return strim(str); } extern char * strnstr(const char *, const char *, size_t); extern char * strpbrk(const char *,const char *); extern char * strsep(char **,const char *); extern __kernel_size_t strspn(const char *,const char *); extern __kernel_size_t strcspn(const char *,const char *); extern int __builtin_memcmp(const void *,const void *,__kernel_size_t); extern char *kstrdup(const char *s, gfp_t gfp); extern char *kstrndup(const char *s, size_t len, gfp_t gfp); extern void *kmemdup(const void *src, size_t len, gfp_t gfp); extern char **argv_split(gfp_t gfp, const char *str, int *argcp); extern void argv_free(char **argv); extern bool sysfs_streq(const char *s1, const char *s2); extern int strtobool(const char *s, bool *res); int vbin_printf(u32 *bin_buf, size_t size, const char *fmt, va_list args); int bstr_printf(char *buf, size_t size, const char *fmt, const u32 *bin_buf); int bprintf(u32 *bin_buf, size_t size, const char *fmt, ...) __attribute__((format(printf,3,4))); extern ssize_t memory_read_from_buffer(void *to, size_t count, loff_t *ppos, const void *from, size_t available); static inline __attribute__((always_inline)) bool strstarts(const char *str, const char *prefix) { return strncmp(str, prefix, strlen(prefix)) == 0; } static inline __attribute__((always_inline)) void clear_page(void *page) { __builtin_memset(page, 0, ((1UL) << 12)); } static inline __attribute__((always_inline)) void copy_page(void *to, void *from) { __builtin_memcpy(to, from, ((1UL) << 12)); } struct page; static inline __attribute__((always_inline)) void clear_user_page(void *page, unsigned long vaddr, struct page *pg) { clear_page(page); } static inline __attribute__((always_inline)) void copy_user_page(void *to, void *from, unsigned long vaddr, struct page *topage) { copy_page(to, from); } extern bool __virt_addr_valid(unsigned long kaddr); static inline __attribute__((always_inline)) __attribute__((__const__)) int get_order(unsigned long size) { int order; size = (size - 1) >> (12 - 1); order = -1; do { size >>= 1; order++; } while (size); return order; } struct task_struct; struct exec_domain; struct task_struct; struct mm_struct; struct vm86_regs { long ebx; long ecx; long edx; long esi; long edi; long ebp; long eax; long __null_ds; long __null_es; long __null_fs; long __null_gs; long orig_eax; long eip; unsigned short cs, __csh; long eflags; long esp; unsigned short ss, __ssh; unsigned short es, __esh; unsigned short ds, __dsh; unsigned short fs, __fsh; unsigned short gs, __gsh; }; struct revectored_struct { unsigned long __map[8]; }; struct vm86_struct { struct vm86_regs regs; unsigned long flags; unsigned long screen_bitmap; unsigned long cpu_type; struct revectored_struct int_revectored; struct revectored_struct int21_revectored; }; struct vm86plus_info_struct { unsigned long force_return_for_pic:1; unsigned long vm86dbg_active:1; unsigned long vm86dbg_TFpendig:1; unsigned long unused:28; unsigned long is_vm86pus:1; unsigned char vm86dbg_intxxtab[32]; }; struct vm86plus_struct { struct vm86_regs regs; unsigned long flags; unsigned long screen_bitmap; unsigned long cpu_type; struct revectored_struct int_revectored; struct revectored_struct int21_revectored; struct vm86plus_info_struct vm86plus; }; extern const char early_idt_handlers[32][10]; struct pt_regs { unsigned long bx; unsigned long cx; unsigned long dx; unsigned long si; unsigned long di; unsigned long bp; unsigned long ax; unsigned long ds; unsigned long es; unsigned long fs; unsigned long gs; unsigned long orig_ax; unsigned long ip; unsigned long cs; unsigned long flags; unsigned long sp; unsigned long ss; }; typedef int (*initcall_t)(void); typedef void (*exitcall_t)(void); extern initcall_t __con_initcall_start[], __con_initcall_end[]; extern initcall_t __security_initcall_start[], __security_initcall_end[]; typedef void (*ctor_fn_t)(void); extern int do_one_initcall(initcall_t fn); extern char __attribute__ ((__section__(".init.data"))) boot_command_line[]; extern char *saved_command_line; extern unsigned int reset_devices; void setup_arch(char **); void prepare_namespace(void); extern void (*late_time_init)(void); extern int initcall_debug; struct cpuinfo_x86; struct task_struct; extern unsigned long profile_pc(struct pt_regs *regs); extern unsigned long convert_ip_to_linear(struct task_struct *child, struct pt_regs *regs); extern void send_sigtrap(struct task_struct *tsk, struct pt_regs *regs, int error_code, int si_code); void signal_fault(struct pt_regs *regs, void *frame, char *where); extern long syscall_trace_enter(struct pt_regs *); extern void syscall_trace_leave(struct pt_regs *); static inline __attribute__((always_inline)) unsigned long regs_return_value(struct pt_regs *regs) { return regs->ax; } static inline __attribute__((always_inline)) int user_mode(struct pt_regs *regs) { return (regs->cs & 0x3) == 0x3; } static inline __attribute__((always_inline)) int user_mode_vm(struct pt_regs *regs) { return ((regs->cs & 0x3) | (regs->flags & 0x00020000)) >= 0x3; } static inline __attribute__((always_inline)) int v8086_mode(struct pt_regs *regs) { return (regs->flags & 0x00020000); } static inline __attribute__((always_inline)) unsigned long kernel_stack_pointer(struct pt_regs *regs) { return (unsigned long)(®s->sp); } static inline __attribute__((always_inline)) unsigned long instruction_pointer(struct pt_regs *regs) { return ((regs)->ip); } static inline __attribute__((always_inline)) void instruction_pointer_set(struct pt_regs *regs, unsigned long val) { (((regs)->ip) = (val)); } static inline __attribute__((always_inline)) unsigned long user_stack_pointer(struct pt_regs *regs) { return ((regs)->sp); } static inline __attribute__((always_inline)) void user_stack_pointer_set(struct pt_regs *regs, unsigned long val) { (((regs)->sp) = (val)); } static inline __attribute__((always_inline)) unsigned long frame_pointer(struct pt_regs *regs) { return ((regs)->bp); } static inline __attribute__((always_inline)) void frame_pointer_set(struct pt_regs *regs, unsigned long val) { (((regs)->bp) = (val)); } extern int regs_query_register_offset(const char *name); extern const char *regs_query_register_name(unsigned int offset); static inline __attribute__((always_inline)) unsigned long regs_get_register(struct pt_regs *regs, unsigned int offset) { if (__builtin_constant_p((((__builtin_constant_p(offset > (__builtin_offsetof(struct pt_regs,ss))) ? !!(offset > (__builtin_offsetof(struct pt_regs,ss))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/ptrace.h", .line = 229, }; ______r = __builtin_expect(!!(offset > (__builtin_offsetof(struct pt_regs,ss))), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(offset > (__builtin_offsetof(struct pt_regs,ss))) ? !!(offset > (__builtin_offsetof(struct pt_regs,ss))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/ptrace.h", .line = 229, }; ______r = __builtin_expect(!!(offset > (__builtin_offsetof(struct pt_regs,ss))), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/ptrace.h", .line = 229, }; ______r = !!(((__builtin_constant_p(offset > (__builtin_offsetof(struct pt_regs,ss))) ? !!(offset > (__builtin_offsetof(struct pt_regs,ss))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/ptrace.h", .line = 229, }; ______r = __builtin_expect(!!(offset > (__builtin_offsetof(struct pt_regs,ss))), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) return 0; return *(unsigned long *)((unsigned long)regs + offset); } static inline __attribute__((always_inline)) int regs_within_kernel_stack(struct pt_regs *regs, unsigned long addr) { return ((addr & ~((((1UL) << 12) << 1) - 1)) == (kernel_stack_pointer(regs) & ~((((1UL) << 12) << 1) - 1))); } static inline __attribute__((always_inline)) unsigned long regs_get_kernel_stack_nth(struct pt_regs *regs, unsigned int n) { unsigned long *addr = (unsigned long *)kernel_stack_pointer(regs); addr += n; if (__builtin_constant_p(((regs_within_kernel_stack(regs, (unsigned long)addr)))) ? !!((regs_within_kernel_stack(regs, (unsigned long)addr))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/ptrace.h", .line = 263, }; ______r = !!((regs_within_kernel_stack(regs, (unsigned long)addr))); ______f.miss_hit[______r]++; ______r; })) return *addr; else return 0; } struct user_desc; extern int do_get_thread_area(struct task_struct *p, int idx, struct user_desc *info); extern int do_set_thread_area(struct task_struct *p, int idx, struct user_desc *info, int can_allocate); struct kernel_vm86_regs { struct pt_regs pt; unsigned short es, __esh; unsigned short ds, __dsh; unsigned short fs, __fsh; unsigned short gs, __gsh; }; struct kernel_vm86_struct { struct kernel_vm86_regs regs; unsigned long flags; unsigned long screen_bitmap; unsigned long cpu_type; struct revectored_struct int_revectored; struct revectored_struct int21_revectored; struct vm86plus_info_struct vm86plus; struct pt_regs *regs32; }; void handle_vm86_fault(struct kernel_vm86_regs *, long); int handle_vm86_trap(struct kernel_vm86_regs *, long, int); struct pt_regs *save_v86_state(struct kernel_vm86_regs *); struct task_struct; void release_vm86_irqs(struct task_struct *); struct math_emu_info { long ___orig_eip; union { struct pt_regs *regs; struct kernel_vm86_regs *vm86; }; }; struct _fpx_sw_bytes { __u32 magic1; __u32 extended_size; __u64 xstate_bv; __u32 xstate_size; __u32 padding[7]; }; struct _fpreg { unsigned short significand[4]; unsigned short exponent; }; struct _fpxreg { unsigned short significand[4]; unsigned short exponent; unsigned short padding[3]; }; struct _xmmreg { unsigned long element[4]; }; struct _fpstate { unsigned long cw; unsigned long sw; unsigned long tag; unsigned long ipoff; unsigned long cssel; unsigned long dataoff; unsigned long datasel; struct _fpreg _st[8]; unsigned short status; unsigned short magic; unsigned long _fxsr_env[6]; unsigned long mxcsr; unsigned long reserved; struct _fpxreg _fxsr_st[8]; struct _xmmreg _xmm[8]; unsigned long padding1[44]; union { unsigned long padding2[12]; struct _fpx_sw_bytes sw_reserved; }; }; struct sigcontext { unsigned short gs, __gsh; unsigned short fs, __fsh; unsigned short es, __esh; unsigned short ds, __dsh; unsigned long di; unsigned long si; unsigned long bp; unsigned long sp; unsigned long bx; unsigned long dx; unsigned long cx; unsigned long ax; unsigned long trapno; unsigned long err; unsigned long ip; unsigned short cs, __csh; unsigned long flags; unsigned long sp_at_signal; unsigned short ss, __ssh; void *fpstate; unsigned long oldmask; unsigned long cr2; }; struct _xsave_hdr { __u64 xstate_bv; __u64 reserved1[2]; __u64 reserved2[5]; }; struct _ymmh_state { __u32 ymmh_space[64]; }; struct _xstate { struct _fpstate fpstate; struct _xsave_hdr xstate_hdr; struct _ymmh_state ymmh; }; extern __attribute__((const, noreturn)) int ____ilog2_NaN(void); static inline __attribute__((always_inline)) __attribute__((const)) int __ilog2_u32(u32 n) { return fls(n) - 1; } static inline __attribute__((always_inline)) __attribute__((const)) int __ilog2_u64(u64 n) { return fls64(n) - 1; } static inline __attribute__((always_inline)) __attribute__((const)) bool is_power_of_2(unsigned long n) { return (n != 0 && ((n & (n - 1)) == 0)); } static inline __attribute__((always_inline)) __attribute__((const)) unsigned long __roundup_pow_of_two(unsigned long n) { return 1UL << fls_long(n - 1); } static inline __attribute__((always_inline)) __attribute__((const)) unsigned long __rounddown_pow_of_two(unsigned long n) { return 1UL << (fls_long(n) - 1); } extern const char linux_banner[]; extern const char linux_proc_banner[]; extern int console_printk[]; static inline __attribute__((always_inline)) void console_silent(void) { (console_printk[0]) = 0; } static inline __attribute__((always_inline)) void console_verbose(void) { if (__builtin_constant_p((((console_printk[0])))) ? !!(((console_printk[0]))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/printk.h", .line = 41, }; ______r = !!(((console_printk[0]))); ______f.miss_hit[______r]++; ______r; })) (console_printk[0]) = 15; } struct va_format { const char *fmt; va_list *va; }; static inline __attribute__((always_inline)) __attribute__ ((format (printf, 1, 2))) int no_printk(const char *fmt, ...) { return 0; } extern __attribute__((regparm(0))) __attribute__ ((format (printf, 1, 2))) void early_printk(const char *fmt, ...); extern int printk_needs_cpu(int cpu); extern void printk_tick(void); __attribute__((regparm(0))) __attribute__ ((format (printf, 1, 0))) int vprintk(const char *fmt, va_list args); __attribute__((regparm(0))) __attribute__ ((format (printf, 1, 2))) __attribute__((__cold__)) int printk(const char *fmt, ...); extern int __printk_ratelimit(const char *func); extern bool printk_timed_ratelimit(unsigned long *caller_jiffies, unsigned int interval_msec); extern int printk_delay_msec; extern int dmesg_restrict; extern int kptr_restrict; void log_buf_kexec_setup(void); void __attribute__ ((__section__(".init.text"))) __attribute__((__cold__)) __attribute__((no_instrument_function)) setup_log_buf(int early); extern void dump_stack(void) __attribute__((__cold__)); enum { DUMP_PREFIX_NONE, DUMP_PREFIX_ADDRESS, DUMP_PREFIX_OFFSET }; extern void hex_dump_to_buffer(const void *buf, size_t len, int rowsize, int groupsize, char *linebuf, size_t linebuflen, bool ascii); extern void print_hex_dump(const char *level, const char *prefix_str, int prefix_type, int rowsize, int groupsize, const void *buf, size_t len, bool ascii); extern void print_hex_dump_bytes(const char *prefix_str, int prefix_type, const void *buf, size_t len); extern long long dynamic_debug_enabled; extern long long dynamic_debug_enabled2; struct _ddebug { const char *modname; const char *function; const char *filename; const char *format; unsigned int lineno:24; unsigned int flags:8; char enabled; } __attribute__((aligned(8))); int ddebug_add_module(struct _ddebug *tab, unsigned int n, const char *modname); static inline __attribute__((always_inline)) int ddebug_remove_module(const char *mod) { return 0; } struct bug_entry { unsigned long bug_addr; const char *file; unsigned short line; unsigned short flags; }; extern void warn_slowpath_fmt(const char *file, const int line, const char *fmt, ...) __attribute__((format(printf, 3, 4))); extern void warn_slowpath_fmt_taint(const char *file, const int line, unsigned taint, const char *fmt, ...) __attribute__((format(printf, 4, 5))); extern void warn_slowpath_null(const char *file, const int line); static inline __attribute__((always_inline)) u64 div_u64_rem(u64 dividend, u32 divisor, u32 *remainder) { union { u64 v64; u32 v32[2]; } d = { dividend }; u32 upper; upper = d.v32[1]; d.v32[1] = 0; if (__builtin_constant_p(((upper >= divisor))) ? !!((upper >= divisor)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/div64.h", .line = 46, }; ______r = !!((upper >= divisor)); ______f.miss_hit[______r]++; ______r; })) { d.v32[1] = upper / divisor; upper %= divisor; } asm ("divl %2" : "=a" (d.v32[0]), "=d" (*remainder) : "rm" (divisor), "0" (d.v32[0]), "1" (upper)); return d.v64; } struct completion; struct pt_regs; struct user; void __might_sleep(const char *file, int line, int preempt_offset); void might_fault(void); extern struct atomic_notifier_head panic_notifier_list; extern long (*panic_blink)(int state); void panic(const char * fmt, ...) __attribute__ ((noreturn, format (printf, 1, 2))) __attribute__((__cold__)); extern void oops_enter(void); extern void oops_exit(void); void print_oops_end_marker(void); extern int oops_may_print(void); void do_exit(long error_code) __attribute__((noreturn)); void complete_and_exit(struct completion *, long) __attribute__((noreturn)); int __attribute__((warn_unused_result)) _kstrtoul(const char *s, unsigned int base, unsigned long *res); int __attribute__((warn_unused_result)) _kstrtol(const char *s, unsigned int base, long *res); int __attribute__((warn_unused_result)) kstrtoull(const char *s, unsigned int base, unsigned long long *res); int __attribute__((warn_unused_result)) kstrtoll(const char *s, unsigned int base, long long *res); static inline __attribute__((always_inline)) int __attribute__((warn_unused_result)) kstrtoul(const char *s, unsigned int base, unsigned long *res) { if (__builtin_constant_p(((sizeof(unsigned long) == sizeof(unsigned long long) && __alignof__(unsigned long) == __alignof__(unsigned long long)))) ? !!((sizeof(unsigned long) == sizeof(unsigned long long) && __alignof__(unsigned long) == __alignof__(unsigned long long))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/kernel.h" , .line = 204 , }; ______r = !!((sizeof(unsigned long) == sizeof(unsigned long long) && __alignof__(unsigned long) == __alignof__(unsigned long long))); ______f.miss_hit[______r]++; ______r; })) return kstrtoull(s, base, (unsigned long long *)res); else return _kstrtoul(s, base, res); } static inline __attribute__((always_inline)) int __attribute__((warn_unused_result)) kstrtol(const char *s, unsigned int base, long *res) { if (__builtin_constant_p(((sizeof(long) == sizeof(long long) && __alignof__(long) == __alignof__(long long)))) ? !!((sizeof(long) == sizeof(long long) && __alignof__(long) == __alignof__(long long))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/kernel.h" , .line = 217 , }; ______r = !!((sizeof(long) == sizeof(long long) && __alignof__(long) == __alignof__(long long))); ______f.miss_hit[______r]++; ______r; })) return kstrtoll(s, base, (long long *)res); else return _kstrtol(s, base, res); } int __attribute__((warn_unused_result)) kstrtouint(const char *s, unsigned int base, unsigned int *res); int __attribute__((warn_unused_result)) kstrtoint(const char *s, unsigned int base, int *res); static inline __attribute__((always_inline)) int __attribute__((warn_unused_result)) kstrtou64(const char *s, unsigned int base, u64 *res) { return kstrtoull(s, base, res); } static inline __attribute__((always_inline)) int __attribute__((warn_unused_result)) kstrtos64(const char *s, unsigned int base, s64 *res) { return kstrtoll(s, base, res); } static inline __attribute__((always_inline)) int __attribute__((warn_unused_result)) kstrtou32(const char *s, unsigned int base, u32 *res) { return kstrtouint(s, base, res); } static inline __attribute__((always_inline)) int __attribute__((warn_unused_result)) kstrtos32(const char *s, unsigned int base, s32 *res) { return kstrtoint(s, base, res); } int __attribute__((warn_unused_result)) kstrtou16(const char *s, unsigned int base, u16 *res); int __attribute__((warn_unused_result)) kstrtos16(const char *s, unsigned int base, s16 *res); int __attribute__((warn_unused_result)) kstrtou8(const char *s, unsigned int base, u8 *res); int __attribute__((warn_unused_result)) kstrtos8(const char *s, unsigned int base, s8 *res); int __attribute__((warn_unused_result)) kstrtoull_from_user(const char *s, size_t count, unsigned int base, unsigned long long *res); int __attribute__((warn_unused_result)) kstrtoll_from_user(const char *s, size_t count, unsigned int base, long long *res); int __attribute__((warn_unused_result)) kstrtoul_from_user(const char *s, size_t count, unsigned int base, unsigned long *res); int __attribute__((warn_unused_result)) kstrtol_from_user(const char *s, size_t count, unsigned int base, long *res); int __attribute__((warn_unused_result)) kstrtouint_from_user(const char *s, size_t count, unsigned int base, unsigned int *res); int __attribute__((warn_unused_result)) kstrtoint_from_user(const char *s, size_t count, unsigned int base, int *res); int __attribute__((warn_unused_result)) kstrtou16_from_user(const char *s, size_t count, unsigned int base, u16 *res); int __attribute__((warn_unused_result)) kstrtos16_from_user(const char *s, size_t count, unsigned int base, s16 *res); int __attribute__((warn_unused_result)) kstrtou8_from_user(const char *s, size_t count, unsigned int base, u8 *res); int __attribute__((warn_unused_result)) kstrtos8_from_user(const char *s, size_t count, unsigned int base, s8 *res); static inline __attribute__((always_inline)) int __attribute__((warn_unused_result)) kstrtou64_from_user(const char *s, size_t count, unsigned int base, u64 *res) { return kstrtoull_from_user(s, count, base, res); } static inline __attribute__((always_inline)) int __attribute__((warn_unused_result)) kstrtos64_from_user(const char *s, size_t count, unsigned int base, s64 *res) { return kstrtoll_from_user(s, count, base, res); } static inline __attribute__((always_inline)) int __attribute__((warn_unused_result)) kstrtou32_from_user(const char *s, size_t count, unsigned int base, u32 *res) { return kstrtouint_from_user(s, count, base, res); } static inline __attribute__((always_inline)) int __attribute__((warn_unused_result)) kstrtos32_from_user(const char *s, size_t count, unsigned int base, s32 *res) { return kstrtoint_from_user(s, count, base, res); } extern unsigned long simple_strtoul(const char *,char **,unsigned int); extern long simple_strtol(const char *,char **,unsigned int); extern unsigned long long simple_strtoull(const char *,char **,unsigned int); extern long long simple_strtoll(const char *,char **,unsigned int); extern int sprintf(char * buf, const char * fmt, ...) __attribute__ ((format (printf, 2, 3))); extern int vsprintf(char *buf, const char *, va_list) __attribute__ ((format (printf, 2, 0))); extern int snprintf(char * buf, size_t size, const char * fmt, ...) __attribute__ ((format (printf, 3, 4))); extern int vsnprintf(char *buf, size_t size, const char *fmt, va_list args) __attribute__ ((format (printf, 3, 0))); extern int scnprintf(char * buf, size_t size, const char * fmt, ...) __attribute__ ((format (printf, 3, 4))); extern int vscnprintf(char *buf, size_t size, const char *fmt, va_list args) __attribute__ ((format (printf, 3, 0))); extern char *kasprintf(gfp_t gfp, const char *fmt, ...) __attribute__ ((format (printf, 2, 3))); extern char *kvasprintf(gfp_t gfp, const char *fmt, va_list args); extern int sscanf(const char *, const char *, ...) __attribute__ ((format (scanf, 2, 3))); extern int vsscanf(const char *, const char *, va_list) __attribute__ ((format (scanf, 2, 0))); extern int get_option(char **str, int *pint); extern char *get_options(const char *str, int nints, int *ints); extern unsigned long long memparse(const char *ptr, char **retptr); extern int core_kernel_text(unsigned long addr); extern int core_kernel_data(unsigned long addr); extern int __kernel_text_address(unsigned long addr); extern int kernel_text_address(unsigned long addr); extern int func_ptr_is_kernel_text(void *ptr); struct pid; extern struct pid *session_of_pgrp(struct pid *pgrp); unsigned long int_sqrt(unsigned long); extern void bust_spinlocks(int yes); extern void wake_up_klogd(void); extern int oops_in_progress; extern int panic_timeout; extern int panic_on_oops; extern int panic_on_unrecovered_nmi; extern int panic_on_io_nmi; extern const char *print_tainted(void); extern void add_taint(unsigned flag); extern int test_taint(unsigned flag); extern unsigned long get_taint(void); extern int root_mountflags; extern bool early_boot_irqs_disabled; extern enum system_states { SYSTEM_BOOTING, SYSTEM_RUNNING, SYSTEM_HALT, SYSTEM_POWER_OFF, SYSTEM_RESTART, SYSTEM_SUSPEND_DISK, } system_state; extern const char hex_asc[]; static inline __attribute__((always_inline)) char *pack_hex_byte(char *buf, u8 byte) { *buf++ = hex_asc[((byte) & 0xf0) >> 4]; *buf++ = hex_asc[((byte) & 0x0f)]; return buf; } extern int hex_to_bin(char ch); extern void hex2bin(u8 *dst, const char *src, size_t count); void tracing_on(void); void tracing_off(void); void tracing_off_permanent(void); int tracing_is_on(void); enum ftrace_dump_mode { DUMP_NONE, DUMP_ALL, DUMP_ORIG, }; extern void tracing_start(void); extern void tracing_stop(void); extern void ftrace_off_permanent(void); static inline __attribute__((always_inline)) void __attribute__ ((format (printf, 1, 2))) ____trace_printk_check_format(const char *fmt, ...) { } extern int __trace_bprintk(unsigned long ip, const char *fmt, ...) __attribute__ ((format (printf, 2, 3))); extern int __trace_printk(unsigned long ip, const char *fmt, ...) __attribute__ ((format (printf, 2, 3))); extern void trace_dump_stack(void); extern int __ftrace_vbprintk(unsigned long ip, const char *fmt, va_list ap); extern int __ftrace_vprintk(unsigned long ip, const char *fmt, va_list ap); extern void ftrace_dump(enum ftrace_dump_mode oops_dump_mode); struct sysinfo; extern int do_sysinfo(struct sysinfo *info); struct sysinfo { long uptime; unsigned long loads[3]; unsigned long totalram; unsigned long freeram; unsigned long sharedram; unsigned long bufferram; unsigned long totalswap; unsigned long freeswap; unsigned short procs; unsigned short pad; unsigned long totalhigh; unsigned long freehigh; unsigned int mem_unit; char _f[20-2*sizeof(long)-sizeof(int)]; }; extern int __build_bug_on_failed; extern void __bad_percpu_size(void); static inline __attribute__((always_inline)) __attribute__((always_inline)) int x86_this_cpu_constant_test_bit(unsigned int nr, const unsigned long *addr) { unsigned long *a = (unsigned long *)addr + nr / 32; return ((1UL << (nr % 32)) & ({ typeof(*a) pfo_ret__; switch (sizeof(*a)) { case 1: asm("mov" "b ""%%""fs"":" "%P" "1"",%0" : "=q" (pfo_ret__) : "m" (*a)); break; case 2: asm("mov" "w ""%%""fs"":" "%P" "1"",%0" : "=r" (pfo_ret__) : "m" (*a)); break; case 4: asm("mov" "l ""%%""fs"":" "%P" "1"",%0" : "=r" (pfo_ret__) : "m" (*a)); break; case 8: asm("mov" "q ""%%""fs"":" "%P" "1"",%0" : "=r" (pfo_ret__) : "m" (*a)); break; default: __bad_percpu_size(); } pfo_ret__; })) != 0; } static inline __attribute__((always_inline)) int x86_this_cpu_variable_test_bit(int nr, const unsigned long *addr) { int oldbit; asm volatile("bt ""%%""fs"":" "%P" "2"",%1\n\t" "sbb %0,%0" : "=r" (oldbit) : "m" (*(unsigned long *)addr), "Ir" (nr)); return oldbit; } extern unsigned long __per_cpu_offset[8]; extern void setup_per_cpu_areas(void); extern __attribute__((section(".data..percpu" ""))) __typeof__(unsigned long) this_cpu_off; struct task_struct; extern __attribute__((section(".data..percpu" ""))) __typeof__(struct task_struct *) current_task; static inline __attribute__((always_inline)) __attribute__((always_inline)) struct task_struct *get_current(void) { return ({ typeof(current_task) pfo_ret__; switch (sizeof(current_task)) { case 1: asm("mov" "b ""%%""fs"":" "%P" "1"",%0" : "=q" (pfo_ret__) : "p" (&(current_task))); break; case 2: asm("mov" "w ""%%""fs"":" "%P" "1"",%0" : "=r" (pfo_ret__) : "p" (&(current_task))); break; case 4: asm("mov" "l ""%%""fs"":" "%P" "1"",%0" : "=r" (pfo_ret__) : "p" (&(current_task))); break; case 8: asm("mov" "q ""%%""fs"":" "%P" "1"",%0" : "=r" (pfo_ret__) : "p" (&(current_task))); break; default: __bad_percpu_size(); } pfo_ret__; }); } extern void __xchg_wrong_size(void); static inline __attribute__((always_inline)) void set_64bit(volatile u64 *ptr, u64 value) { u32 low = value; u32 high = value >> 32; u64 prev = *ptr; asm volatile("\n1:\t" ".section .smp_locks,\"a\"\n" ".balign 4\n" ".long 671f - .\n" ".previous\n" "671:" "\n\tlock; " "cmpxchg8b %0\n\t" "jnz 1b" : "=m" (*ptr), "+A" (prev) : "b" (low), "c" (high) : "memory"); } extern void __cmpxchg_wrong_size(void); static inline __attribute__((always_inline)) u64 __cmpxchg64(volatile u64 *ptr, u64 old, u64 new) { u64 prev; asm volatile(".section .smp_locks,\"a\"\n" ".balign 4\n" ".long 671f - .\n" ".previous\n" "671:" "\n\tlock; " "cmpxchg8b %1" : "=A" (prev), "+m" (*ptr) : "b" ((u32)new), "c" ((u32)(new >> 32)), "0" (old) : "memory"); return prev; } static inline __attribute__((always_inline)) u64 __cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new) { u64 prev; asm volatile("cmpxchg8b %1" : "=A" (prev), "+m" (*ptr) : "b" ((u32)new), "c" ((u32)(new >> 32)), "0" (old) : "memory"); return prev; } extern const unsigned char * const *ideal_nops; extern void arch_init_ideal_nops(void); static inline __attribute__((always_inline)) unsigned long native_save_fl(void) { unsigned long flags; asm volatile("# __raw_save_flags\n\t" "pushf ; pop %0" : "=rm" (flags) : : "memory"); return flags; } static inline __attribute__((always_inline)) void native_restore_fl(unsigned long flags) { asm volatile("push %0 ; popf" : :"g" (flags) :"memory", "cc"); } static inline __attribute__((always_inline)) void native_irq_disable(void) { asm volatile("cli": : :"memory"); } static inline __attribute__((always_inline)) void native_irq_enable(void) { asm volatile("sti": : :"memory"); } static inline __attribute__((always_inline)) void native_safe_halt(void) { asm volatile("sti; hlt": : :"memory"); } static inline __attribute__((always_inline)) void native_halt(void) { asm volatile("hlt": : :"memory"); } typedef u64 pteval_t; typedef u64 pmdval_t; typedef u64 pudval_t; typedef u64 pgdval_t; typedef u64 pgprotval_t; typedef union { struct { unsigned long pte_low, pte_high; }; pteval_t pte; } pte_t; extern bool __vmalloc_start_set; typedef struct pgprot { pgprotval_t pgprot; } pgprot_t; typedef struct { pgdval_t pgd; } pgd_t; static inline __attribute__((always_inline)) pgd_t native_make_pgd(pgdval_t val) { return (pgd_t) { val }; } static inline __attribute__((always_inline)) pgdval_t native_pgd_val(pgd_t pgd) { return pgd.pgd; } static inline __attribute__((always_inline)) pgdval_t pgd_flags(pgd_t pgd) { return native_pgd_val(pgd) & (~((pteval_t)(((signed long)(~(((1UL) << 12)-1))) & ((phys_addr_t)((1ULL << 44) - 1))))); } typedef struct { pgd_t pgd; } pud_t; static inline __attribute__((always_inline)) int pgd_none(pgd_t pgd) { return 0; } static inline __attribute__((always_inline)) int pgd_bad(pgd_t pgd) { return 0; } static inline __attribute__((always_inline)) int pgd_present(pgd_t pgd) { return 1; } static inline __attribute__((always_inline)) void pgd_clear(pgd_t *pgd) { } static inline __attribute__((always_inline)) pud_t * pud_offset(pgd_t * pgd, unsigned long address) { return (pud_t *)pgd; } static inline __attribute__((always_inline)) pudval_t native_pud_val(pud_t pud) { return native_pgd_val(pud.pgd); } typedef struct { pmdval_t pmd; } pmd_t; static inline __attribute__((always_inline)) pmd_t native_make_pmd(pmdval_t val) { return (pmd_t) { val }; } static inline __attribute__((always_inline)) pmdval_t native_pmd_val(pmd_t pmd) { return pmd.pmd; } static inline __attribute__((always_inline)) pudval_t pud_flags(pud_t pud) { return native_pud_val(pud) & (~((pteval_t)(((signed long)(~(((1UL) << 12)-1))) & ((phys_addr_t)((1ULL << 44) - 1))))); } static inline __attribute__((always_inline)) pmdval_t pmd_flags(pmd_t pmd) { return native_pmd_val(pmd) & (~((pteval_t)(((signed long)(~(((1UL) << 12)-1))) & ((phys_addr_t)((1ULL << 44) - 1))))); } static inline __attribute__((always_inline)) pte_t native_make_pte(pteval_t val) { return (pte_t) { .pte = val }; } static inline __attribute__((always_inline)) pteval_t native_pte_val(pte_t pte) { return pte.pte; } static inline __attribute__((always_inline)) pteval_t pte_flags(pte_t pte) { return native_pte_val(pte) & (~((pteval_t)(((signed long)(~(((1UL) << 12)-1))) & ((phys_addr_t)((1ULL << 44) - 1))))); } typedef struct page *pgtable_t; extern pteval_t __supported_pte_mask; extern void set_nx(void); extern int nx_enabled; extern pgprot_t pgprot_writecombine(pgprot_t prot); struct file; pgprot_t phys_mem_access_prot(struct file *file, unsigned long pfn, unsigned long size, pgprot_t vma_prot); int phys_mem_access_prot_allowed(struct file *file, unsigned long pfn, unsigned long size, pgprot_t *vma_prot); void set_pte_vaddr(unsigned long vaddr, pte_t pte); extern void native_pagetable_reserve(u64 start, u64 end); extern void native_pagetable_setup_start(pgd_t *base); extern void native_pagetable_setup_done(pgd_t *base); struct seq_file; extern void arch_report_meminfo(struct seq_file *m); enum { PG_LEVEL_NONE, PG_LEVEL_4K, PG_LEVEL_2M, PG_LEVEL_1G, PG_LEVEL_NUM }; extern void update_page_count(int level, unsigned long pages); extern pte_t *lookup_address(unsigned long address, unsigned int *level); struct desc_struct { union { struct { unsigned int a; unsigned int b; }; struct { u16 limit0; u16 base0; unsigned base1: 8, type: 4, s: 1, dpl: 2, p: 1; unsigned limit: 4, avl: 1, l: 1, d: 1, g: 1, base2: 8; }; }; } __attribute__((packed)); enum { GATE_INTERRUPT = 0xE, GATE_TRAP = 0xF, GATE_CALL = 0xC, GATE_TASK = 0x5, }; struct gate_struct64 { u16 offset_low; u16 segment; unsigned ist : 3, zero0 : 5, type : 5, dpl : 2, p : 1; u16 offset_middle; u32 offset_high; u32 zero1; } __attribute__((packed)); enum { DESC_TSS = 0x9, DESC_LDT = 0x2, DESCTYPE_S = 0x10, }; struct ldttss_desc64 { u16 limit0; u16 base0; unsigned base1 : 8, type : 5, dpl : 2, p : 1; unsigned limit1 : 4, zero0 : 3, g : 1, base2 : 8; u32 base3; u32 zero1; } __attribute__((packed)); typedef struct desc_struct gate_desc; typedef struct desc_struct ldt_desc; typedef struct desc_struct tss_desc; struct desc_ptr { unsigned short size; unsigned long address; } __attribute__((packed)) ; enum km_type { KM_BOUNCE_READ, KM_SKB_SUNRPC_DATA, KM_SKB_DATA_SOFTIRQ, KM_USER0, KM_USER1, KM_BIO_SRC_IRQ, KM_BIO_DST_IRQ, KM_PTE0, KM_PTE1, KM_IRQ0, KM_IRQ1, KM_SOFTIRQ0, KM_SOFTIRQ1, KM_SYNC_ICACHE, KM_SYNC_DCACHE, KM_UML_USERCOPY, KM_IRQ_PTE, KM_NMI, KM_NMI_PTE, KM_KDB, KM_TYPE_NR }; struct page; struct thread_struct; struct desc_ptr; struct tss_struct; struct mm_struct; struct desc_struct; struct task_struct; struct cpumask; struct paravirt_callee_save { void *func; }; struct pv_info { unsigned int kernel_rpl; int shared_kernel_pmd; int paravirt_enabled; const char *name; }; struct pv_init_ops { unsigned (*patch)(u8 type, u16 clobber, void *insnbuf, unsigned long addr, unsigned len); }; struct pv_lazy_ops { void (*enter)(void); void (*leave)(void); }; struct pv_time_ops { unsigned long long (*sched_clock)(void); unsigned long (*get_tsc_khz)(void); }; struct pv_cpu_ops { unsigned long (*get_debugreg)(int regno); void (*set_debugreg)(int regno, unsigned long value); void (*clts)(void); unsigned long (*read_cr0)(void); void (*write_cr0)(unsigned long); unsigned long (*read_cr4_safe)(void); unsigned long (*read_cr4)(void); void (*write_cr4)(unsigned long); void (*load_tr_desc)(void); void (*load_gdt)(const struct desc_ptr *); void (*load_idt)(const struct desc_ptr *); void (*store_gdt)(struct desc_ptr *); void (*store_idt)(struct desc_ptr *); void (*set_ldt)(const void *desc, unsigned entries); unsigned long (*store_tr)(void); void (*load_tls)(struct thread_struct *t, unsigned int cpu); void (*write_ldt_entry)(struct desc_struct *ldt, int entrynum, const void *desc); void (*write_gdt_entry)(struct desc_struct *, int entrynum, const void *desc, int size); void (*write_idt_entry)(gate_desc *, int entrynum, const gate_desc *gate); void (*alloc_ldt)(struct desc_struct *ldt, unsigned entries); void (*free_ldt)(struct desc_struct *ldt, unsigned entries); void (*load_sp0)(struct tss_struct *tss, struct thread_struct *t); void (*set_iopl_mask)(unsigned mask); void (*wbinvd)(void); void (*io_delay)(void); void (*cpuid)(unsigned int *eax, unsigned int *ebx, unsigned int *ecx, unsigned int *edx); u64 (*read_msr)(unsigned int msr, int *err); int (*rdmsr_regs)(u32 *regs); int (*write_msr)(unsigned int msr, unsigned low, unsigned high); int (*wrmsr_regs)(u32 *regs); u64 (*read_tsc)(void); u64 (*read_pmc)(int counter); unsigned long long (*read_tscp)(unsigned int *aux); void (*irq_enable_sysexit)(void); void (*usergs_sysret64)(void); void (*usergs_sysret32)(void); void (*iret)(void); void (*swapgs)(void); void (*start_context_switch)(struct task_struct *prev); void (*end_context_switch)(struct task_struct *next); }; struct pv_irq_ops { struct paravirt_callee_save save_fl; struct paravirt_callee_save restore_fl; struct paravirt_callee_save irq_disable; struct paravirt_callee_save irq_enable; void (*safe_halt)(void); void (*halt)(void); }; struct pv_apic_ops { void (*startup_ipi_hook)(int phys_apicid, unsigned long start_eip, unsigned long start_esp); }; struct pv_mmu_ops { unsigned long (*read_cr2)(void); void (*write_cr2)(unsigned long); unsigned long (*read_cr3)(void); void (*write_cr3)(unsigned long); void (*activate_mm)(struct mm_struct *prev, struct mm_struct *next); void (*dup_mmap)(struct mm_struct *oldmm, struct mm_struct *mm); void (*exit_mmap)(struct mm_struct *mm); void (*flush_tlb_user)(void); void (*flush_tlb_kernel)(void); void (*flush_tlb_single)(unsigned long addr); void (*flush_tlb_others)(const struct cpumask *cpus, struct mm_struct *mm, unsigned long va); int (*pgd_alloc)(struct mm_struct *mm); void (*pgd_free)(struct mm_struct *mm, pgd_t *pgd); void (*alloc_pte)(struct mm_struct *mm, unsigned long pfn); void (*alloc_pmd)(struct mm_struct *mm, unsigned long pfn); void (*alloc_pud)(struct mm_struct *mm, unsigned long pfn); void (*release_pte)(unsigned long pfn); void (*release_pmd)(unsigned long pfn); void (*release_pud)(unsigned long pfn); void (*set_pte)(pte_t *ptep, pte_t pteval); void (*set_pte_at)(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pteval); void (*set_pmd)(pmd_t *pmdp, pmd_t pmdval); void (*set_pmd_at)(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp, pmd_t pmdval); void (*pte_update)(struct mm_struct *mm, unsigned long addr, pte_t *ptep); void (*pte_update_defer)(struct mm_struct *mm, unsigned long addr, pte_t *ptep); void (*pmd_update)(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp); void (*pmd_update_defer)(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp); pte_t (*ptep_modify_prot_start)(struct mm_struct *mm, unsigned long addr, pte_t *ptep); void (*ptep_modify_prot_commit)(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte); struct paravirt_callee_save pte_val; struct paravirt_callee_save make_pte; struct paravirt_callee_save pgd_val; struct paravirt_callee_save make_pgd; void (*set_pte_atomic)(pte_t *ptep, pte_t pteval); void (*pte_clear)(struct mm_struct *mm, unsigned long addr, pte_t *ptep); void (*pmd_clear)(pmd_t *pmdp); void (*set_pud)(pud_t *pudp, pud_t pudval); struct paravirt_callee_save pmd_val; struct paravirt_callee_save make_pmd; struct pv_lazy_ops lazy_mode; void (*set_fixmap)(unsigned idx, phys_addr_t phys, pgprot_t flags); }; struct arch_spinlock; struct pv_lock_ops { int (*spin_is_locked)(struct arch_spinlock *lock); int (*spin_is_contended)(struct arch_spinlock *lock); void (*spin_lock)(struct arch_spinlock *lock); void (*spin_lock_flags)(struct arch_spinlock *lock, unsigned long flags); int (*spin_trylock)(struct arch_spinlock *lock); void (*spin_unlock)(struct arch_spinlock *lock); }; struct paravirt_patch_template { struct pv_init_ops pv_init_ops; struct pv_time_ops pv_time_ops; struct pv_cpu_ops pv_cpu_ops; struct pv_irq_ops pv_irq_ops; struct pv_apic_ops pv_apic_ops; struct pv_mmu_ops pv_mmu_ops; struct pv_lock_ops pv_lock_ops; }; extern struct pv_info pv_info; extern struct pv_init_ops pv_init_ops; extern struct pv_time_ops pv_time_ops; extern struct pv_cpu_ops pv_cpu_ops; extern struct pv_irq_ops pv_irq_ops; extern struct pv_apic_ops pv_apic_ops; extern struct pv_mmu_ops pv_mmu_ops; extern struct pv_lock_ops pv_lock_ops; unsigned paravirt_patch_nop(void); unsigned paravirt_patch_ident_32(void *insnbuf, unsigned len); unsigned paravirt_patch_ident_64(void *insnbuf, unsigned len); unsigned paravirt_patch_ignore(unsigned len); unsigned paravirt_patch_call(void *insnbuf, const void *target, u16 tgt_clobbers, unsigned long addr, u16 site_clobbers, unsigned len); unsigned paravirt_patch_jmp(void *insnbuf, const void *target, unsigned long addr, unsigned len); unsigned paravirt_patch_default(u8 type, u16 clobbers, void *insnbuf, unsigned long addr, unsigned len); unsigned paravirt_patch_insns(void *insnbuf, unsigned len, const char *start, const char *end); unsigned native_patch(u8 type, u16 clobbers, void *ibuf, unsigned long addr, unsigned len); int paravirt_disable_iospace(void); enum paravirt_lazy_mode { PARAVIRT_LAZY_NONE, PARAVIRT_LAZY_MMU, PARAVIRT_LAZY_CPU, }; enum paravirt_lazy_mode paravirt_get_lazy_mode(void); void paravirt_start_context_switch(struct task_struct *prev); void paravirt_end_context_switch(struct task_struct *next); void paravirt_enter_lazy_mmu(void); void paravirt_leave_lazy_mmu(void); void _paravirt_nop(void); u32 _paravirt_ident_32(u32); u64 _paravirt_ident_64(u64); struct paravirt_patch_site { u8 *instr; u8 instrtype; u8 len; u16 clobbers; }; extern struct paravirt_patch_site __parainstructions[], __parainstructions_end[]; extern int __bitmap_empty(const unsigned long *bitmap, int bits); extern int __bitmap_full(const unsigned long *bitmap, int bits); extern int __bitmap_equal(const unsigned long *bitmap1, const unsigned long *bitmap2, int bits); extern void __bitmap_complement(unsigned long *dst, const unsigned long *src, int bits); extern void __bitmap_shift_right(unsigned long *dst, const unsigned long *src, int shift, int bits); extern void __bitmap_shift_left(unsigned long *dst, const unsigned long *src, int shift, int bits); extern int __bitmap_and(unsigned long *dst, const unsigned long *bitmap1, const unsigned long *bitmap2, int bits); extern void __bitmap_or(unsigned long *dst, const unsigned long *bitmap1, const unsigned long *bitmap2, int bits); extern void __bitmap_xor(unsigned long *dst, const unsigned long *bitmap1, const unsigned long *bitmap2, int bits); extern int __bitmap_andnot(unsigned long *dst, const unsigned long *bitmap1, const unsigned long *bitmap2, int bits); extern int __bitmap_intersects(const unsigned long *bitmap1, const unsigned long *bitmap2, int bits); extern int __bitmap_subset(const unsigned long *bitmap1, const unsigned long *bitmap2, int bits); extern int __bitmap_weight(const unsigned long *bitmap, int bits); extern void bitmap_set(unsigned long *map, int i, int len); extern void bitmap_clear(unsigned long *map, int start, int nr); extern unsigned long bitmap_find_next_zero_area(unsigned long *map, unsigned long size, unsigned long start, unsigned int nr, unsigned long align_mask); extern int bitmap_scnprintf(char *buf, unsigned int len, const unsigned long *src, int nbits); extern int __bitmap_parse(const char *buf, unsigned int buflen, int is_user, unsigned long *dst, int nbits); extern int bitmap_parse_user(const char *ubuf, unsigned int ulen, unsigned long *dst, int nbits); extern int bitmap_scnlistprintf(char *buf, unsigned int len, const unsigned long *src, int nbits); extern int bitmap_parselist(const char *buf, unsigned long *maskp, int nmaskbits); extern int bitmap_parselist_user(const char *ubuf, unsigned int ulen, unsigned long *dst, int nbits); extern void bitmap_remap(unsigned long *dst, const unsigned long *src, const unsigned long *old, const unsigned long *new, int bits); extern int bitmap_bitremap(int oldbit, const unsigned long *old, const unsigned long *new, int bits); extern void bitmap_onto(unsigned long *dst, const unsigned long *orig, const unsigned long *relmap, int bits); extern void bitmap_fold(unsigned long *dst, const unsigned long *orig, int sz, int bits); extern int bitmap_find_free_region(unsigned long *bitmap, int bits, int order); extern void bitmap_release_region(unsigned long *bitmap, int pos, int order); extern int bitmap_allocate_region(unsigned long *bitmap, int pos, int order); extern void bitmap_copy_le(void *dst, const unsigned long *src, int nbits); static inline __attribute__((always_inline)) void bitmap_zero(unsigned long *dst, int nbits) { if (__builtin_constant_p((((__builtin_constant_p(nbits) && (nbits) <= 32)))) ? !!(((__builtin_constant_p(nbits) && (nbits) <= 32))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/bitmap.h", .line = 159, }; ______r = !!(((__builtin_constant_p(nbits) && (nbits) <= 32))); ______f.miss_hit[______r]++; ______r; })) *dst = 0UL; else { int len = (((nbits) + (8 * sizeof(long)) - 1) / (8 * sizeof(long))) * sizeof(unsigned long); __builtin_memset(dst, 0, len); } } static inline __attribute__((always_inline)) void bitmap_fill(unsigned long *dst, int nbits) { size_t nlongs = (((nbits) + (8 * sizeof(long)) - 1) / (8 * sizeof(long))); if (__builtin_constant_p(((!(__builtin_constant_p(nbits) && (nbits) <= 32)))) ? !!((!(__builtin_constant_p(nbits) && (nbits) <= 32))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/bitmap.h", .line = 170, }; ______r = !!((!(__builtin_constant_p(nbits) && (nbits) <= 32))); ______f.miss_hit[______r]++; ______r; })) { int len = (nlongs - 1) * sizeof(unsigned long); __builtin_memset(dst, 0xff, len); } dst[nlongs - 1] = ( ((nbits) % 32) ? (1UL<<((nbits) % 32))-1 : ~0UL ); } static inline __attribute__((always_inline)) void bitmap_copy(unsigned long *dst, const unsigned long *src, int nbits) { if (__builtin_constant_p((((__builtin_constant_p(nbits) && (nbits) <= 32)))) ? !!(((__builtin_constant_p(nbits) && (nbits) <= 32))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/bitmap.h", .line = 180, }; ______r = !!(((__builtin_constant_p(nbits) && (nbits) <= 32))); ______f.miss_hit[______r]++; ______r; })) *dst = *src; else { int len = (((nbits) + (8 * sizeof(long)) - 1) / (8 * sizeof(long))) * sizeof(unsigned long); __builtin_memcpy(dst, src, len); } } static inline __attribute__((always_inline)) int bitmap_and(unsigned long *dst, const unsigned long *src1, const unsigned long *src2, int nbits) { if (__builtin_constant_p((((__builtin_constant_p(nbits) && (nbits) <= 32)))) ? !!(((__builtin_constant_p(nbits) && (nbits) <= 32))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/bitmap.h", .line = 191, }; ______r = !!(((__builtin_constant_p(nbits) && (nbits) <= 32))); ______f.miss_hit[______r]++; ______r; })) return (*dst = *src1 & *src2) != 0; return __bitmap_and(dst, src1, src2, nbits); } static inline __attribute__((always_inline)) void bitmap_or(unsigned long *dst, const unsigned long *src1, const unsigned long *src2, int nbits) { if (__builtin_constant_p((((__builtin_constant_p(nbits) && (nbits) <= 32)))) ? !!(((__builtin_constant_p(nbits) && (nbits) <= 32))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/bitmap.h", .line = 199, }; ______r = !!(((__builtin_constant_p(nbits) && (nbits) <= 32))); ______f.miss_hit[______r]++; ______r; })) *dst = *src1 | *src2; else __bitmap_or(dst, src1, src2, nbits); } static inline __attribute__((always_inline)) void bitmap_xor(unsigned long *dst, const unsigned long *src1, const unsigned long *src2, int nbits) { if (__builtin_constant_p((((__builtin_constant_p(nbits) && (nbits) <= 32)))) ? !!(((__builtin_constant_p(nbits) && (nbits) <= 32))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/bitmap.h", .line = 208, }; ______r = !!(((__builtin_constant_p(nbits) && (nbits) <= 32))); ______f.miss_hit[______r]++; ______r; })) *dst = *src1 ^ *src2; else __bitmap_xor(dst, src1, src2, nbits); } static inline __attribute__((always_inline)) int bitmap_andnot(unsigned long *dst, const unsigned long *src1, const unsigned long *src2, int nbits) { if (__builtin_constant_p((((__builtin_constant_p(nbits) && (nbits) <= 32)))) ? !!(((__builtin_constant_p(nbits) && (nbits) <= 32))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/bitmap.h", .line = 217, }; ______r = !!(((__builtin_constant_p(nbits) && (nbits) <= 32))); ______f.miss_hit[______r]++; ______r; })) return (*dst = *src1 & ~(*src2)) != 0; return __bitmap_andnot(dst, src1, src2, nbits); } static inline __attribute__((always_inline)) void bitmap_complement(unsigned long *dst, const unsigned long *src, int nbits) { if (__builtin_constant_p((((__builtin_constant_p(nbits) && (nbits) <= 32)))) ? !!(((__builtin_constant_p(nbits) && (nbits) <= 32))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/bitmap.h", .line = 225, }; ______r = !!(((__builtin_constant_p(nbits) && (nbits) <= 32))); ______f.miss_hit[______r]++; ______r; })) *dst = ~(*src) & ( ((nbits) % 32) ? (1UL<<((nbits) % 32))-1 : ~0UL ); else __bitmap_complement(dst, src, nbits); } static inline __attribute__((always_inline)) int bitmap_equal(const unsigned long *src1, const unsigned long *src2, int nbits) { if (__builtin_constant_p((((__builtin_constant_p(nbits) && (nbits) <= 32)))) ? !!(((__builtin_constant_p(nbits) && (nbits) <= 32))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/bitmap.h", .line = 234, }; ______r = !!(((__builtin_constant_p(nbits) && (nbits) <= 32))); ______f.miss_hit[______r]++; ______r; })) return ! ((*src1 ^ *src2) & ( ((nbits) % 32) ? (1UL<<((nbits) % 32))-1 : ~0UL )); else return __bitmap_equal(src1, src2, nbits); } static inline __attribute__((always_inline)) int bitmap_intersects(const unsigned long *src1, const unsigned long *src2, int nbits) { if (__builtin_constant_p((((__builtin_constant_p(nbits) && (nbits) <= 32)))) ? !!(((__builtin_constant_p(nbits) && (nbits) <= 32))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/bitmap.h", .line = 243, }; ______r = !!(((__builtin_constant_p(nbits) && (nbits) <= 32))); ______f.miss_hit[______r]++; ______r; })) return ((*src1 & *src2) & ( ((nbits) % 32) ? (1UL<<((nbits) % 32))-1 : ~0UL )) != 0; else return __bitmap_intersects(src1, src2, nbits); } static inline __attribute__((always_inline)) int bitmap_subset(const unsigned long *src1, const unsigned long *src2, int nbits) { if (__builtin_constant_p((((__builtin_constant_p(nbits) && (nbits) <= 32)))) ? !!(((__builtin_constant_p(nbits) && (nbits) <= 32))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/bitmap.h", .line = 252, }; ______r = !!(((__builtin_constant_p(nbits) && (nbits) <= 32))); ______f.miss_hit[______r]++; ______r; })) return ! ((*src1 & ~(*src2)) & ( ((nbits) % 32) ? (1UL<<((nbits) % 32))-1 : ~0UL )); else return __bitmap_subset(src1, src2, nbits); } static inline __attribute__((always_inline)) int bitmap_empty(const unsigned long *src, int nbits) { if (__builtin_constant_p((((__builtin_constant_p(nbits) && (nbits) <= 32)))) ? !!(((__builtin_constant_p(nbits) && (nbits) <= 32))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/bitmap.h", .line = 260, }; ______r = !!(((__builtin_constant_p(nbits) && (nbits) <= 32))); ______f.miss_hit[______r]++; ______r; })) return ! (*src & ( ((nbits) % 32) ? (1UL<<((nbits) % 32))-1 : ~0UL )); else return __bitmap_empty(src, nbits); } static inline __attribute__((always_inline)) int bitmap_full(const unsigned long *src, int nbits) { if (__builtin_constant_p((((__builtin_constant_p(nbits) && (nbits) <= 32)))) ? !!(((__builtin_constant_p(nbits) && (nbits) <= 32))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/bitmap.h", .line = 268, }; ______r = !!(((__builtin_constant_p(nbits) && (nbits) <= 32))); ______f.miss_hit[______r]++; ______r; })) return ! (~(*src) & ( ((nbits) % 32) ? (1UL<<((nbits) % 32))-1 : ~0UL )); else return __bitmap_full(src, nbits); } static inline __attribute__((always_inline)) int bitmap_weight(const unsigned long *src, int nbits) { if (__builtin_constant_p((((__builtin_constant_p(nbits) && (nbits) <= 32)))) ? !!(((__builtin_constant_p(nbits) && (nbits) <= 32))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/bitmap.h", .line = 276, }; ______r = !!(((__builtin_constant_p(nbits) && (nbits) <= 32))); ______f.miss_hit[______r]++; ______r; })) return hweight_long(*src & ( ((nbits) % 32) ? (1UL<<((nbits) % 32))-1 : ~0UL )); return __bitmap_weight(src, nbits); } static inline __attribute__((always_inline)) void bitmap_shift_right(unsigned long *dst, const unsigned long *src, int n, int nbits) { if (__builtin_constant_p((((__builtin_constant_p(nbits) && (nbits) <= 32)))) ? !!(((__builtin_constant_p(nbits) && (nbits) <= 32))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/bitmap.h", .line = 284, }; ______r = !!(((__builtin_constant_p(nbits) && (nbits) <= 32))); ______f.miss_hit[______r]++; ______r; })) *dst = *src >> n; else __bitmap_shift_right(dst, src, n, nbits); } static inline __attribute__((always_inline)) void bitmap_shift_left(unsigned long *dst, const unsigned long *src, int n, int nbits) { if (__builtin_constant_p((((__builtin_constant_p(nbits) && (nbits) <= 32)))) ? !!(((__builtin_constant_p(nbits) && (nbits) <= 32))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/bitmap.h", .line = 293, }; ______r = !!(((__builtin_constant_p(nbits) && (nbits) <= 32))); ______f.miss_hit[______r]++; ______r; })) *dst = (*src << n) & ( ((nbits) % 32) ? (1UL<<((nbits) % 32))-1 : ~0UL ); else __bitmap_shift_left(dst, src, n, nbits); } static inline __attribute__((always_inline)) int bitmap_parse(const char *buf, unsigned int buflen, unsigned long *maskp, int nmaskbits) { return __bitmap_parse(buf, buflen, 0, maskp, nmaskbits); } typedef struct cpumask { unsigned long bits[(((8) + (8 * sizeof(long)) - 1) / (8 * sizeof(long)))]; } cpumask_t; extern int nr_cpu_ids; extern const struct cpumask *const cpu_possible_mask; extern const struct cpumask *const cpu_online_mask; extern const struct cpumask *const cpu_present_mask; extern const struct cpumask *const cpu_active_mask; static inline __attribute__((always_inline)) unsigned int cpumask_check(unsigned int cpu) { return cpu; } static inline __attribute__((always_inline)) unsigned int cpumask_first(const struct cpumask *srcp) { return find_first_bit(((srcp)->bits), 8); } static inline __attribute__((always_inline)) unsigned int cpumask_next(int n, const struct cpumask *srcp) { if (__builtin_constant_p(((n != -1))) ? !!((n != -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/cpumask.h", .line = 172, }; ______r = !!((n != -1)); ______f.miss_hit[______r]++; ______r; })) cpumask_check(n); return find_next_bit(((srcp)->bits), 8, n+1); } static inline __attribute__((always_inline)) unsigned int cpumask_next_zero(int n, const struct cpumask *srcp) { if (__builtin_constant_p(((n != -1))) ? !!((n != -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/cpumask.h", .line = 187, }; ______r = !!((n != -1)); ______f.miss_hit[______r]++; ______r; })) cpumask_check(n); return find_next_zero_bit(((srcp)->bits), 8, n+1); } int cpumask_next_and(int n, const struct cpumask *, const struct cpumask *); int cpumask_any_but(const struct cpumask *mask, unsigned int cpu); static inline __attribute__((always_inline)) void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp) { set_bit(cpumask_check(cpu), ((dstp)->bits)); } static inline __attribute__((always_inline)) void cpumask_clear_cpu(int cpu, struct cpumask *dstp) { clear_bit(cpumask_check(cpu), ((dstp)->bits)); } static inline __attribute__((always_inline)) int cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask) { return test_and_set_bit(cpumask_check(cpu), ((cpumask)->bits)); } static inline __attribute__((always_inline)) int cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask) { return test_and_clear_bit(cpumask_check(cpu), ((cpumask)->bits)); } static inline __attribute__((always_inline)) void cpumask_setall(struct cpumask *dstp) { bitmap_fill(((dstp)->bits), 8); } static inline __attribute__((always_inline)) void cpumask_clear(struct cpumask *dstp) { bitmap_zero(((dstp)->bits), 8); } static inline __attribute__((always_inline)) int cpumask_and(struct cpumask *dstp, const struct cpumask *src1p, const struct cpumask *src2p) { return bitmap_and(((dstp)->bits), ((src1p)->bits), ((src2p)->bits), 8); } static inline __attribute__((always_inline)) void cpumask_or(struct cpumask *dstp, const struct cpumask *src1p, const struct cpumask *src2p) { bitmap_or(((dstp)->bits), ((src1p)->bits), ((src2p)->bits), 8); } static inline __attribute__((always_inline)) void cpumask_xor(struct cpumask *dstp, const struct cpumask *src1p, const struct cpumask *src2p) { bitmap_xor(((dstp)->bits), ((src1p)->bits), ((src2p)->bits), 8); } static inline __attribute__((always_inline)) int cpumask_andnot(struct cpumask *dstp, const struct cpumask *src1p, const struct cpumask *src2p) { return bitmap_andnot(((dstp)->bits), ((src1p)->bits), ((src2p)->bits), 8); } static inline __attribute__((always_inline)) void cpumask_complement(struct cpumask *dstp, const struct cpumask *srcp) { bitmap_complement(((dstp)->bits), ((srcp)->bits), 8); } static inline __attribute__((always_inline)) bool cpumask_equal(const struct cpumask *src1p, const struct cpumask *src2p) { return bitmap_equal(((src1p)->bits), ((src2p)->bits), 8); } static inline __attribute__((always_inline)) bool cpumask_intersects(const struct cpumask *src1p, const struct cpumask *src2p) { return bitmap_intersects(((src1p)->bits), ((src2p)->bits), 8); } static inline __attribute__((always_inline)) int cpumask_subset(const struct cpumask *src1p, const struct cpumask *src2p) { return bitmap_subset(((src1p)->bits), ((src2p)->bits), 8); } static inline __attribute__((always_inline)) bool cpumask_empty(const struct cpumask *srcp) { return bitmap_empty(((srcp)->bits), 8); } static inline __attribute__((always_inline)) bool cpumask_full(const struct cpumask *srcp) { return bitmap_full(((srcp)->bits), 8); } static inline __attribute__((always_inline)) unsigned int cpumask_weight(const struct cpumask *srcp) { return bitmap_weight(((srcp)->bits), 8); } static inline __attribute__((always_inline)) void cpumask_shift_right(struct cpumask *dstp, const struct cpumask *srcp, int n) { bitmap_shift_right(((dstp)->bits), ((srcp)->bits), n, 8); } static inline __attribute__((always_inline)) void cpumask_shift_left(struct cpumask *dstp, const struct cpumask *srcp, int n) { bitmap_shift_left(((dstp)->bits), ((srcp)->bits), n, 8); } static inline __attribute__((always_inline)) void cpumask_copy(struct cpumask *dstp, const struct cpumask *srcp) { bitmap_copy(((dstp)->bits), ((srcp)->bits), 8); } static inline __attribute__((always_inline)) int cpumask_scnprintf(char *buf, int len, const struct cpumask *srcp) { return bitmap_scnprintf(buf, len, ((srcp)->bits), 8); } static inline __attribute__((always_inline)) int cpumask_parse_user(const char *buf, int len, struct cpumask *dstp) { return bitmap_parse_user(buf, len, ((dstp)->bits), 8); } static inline __attribute__((always_inline)) int cpumask_parselist_user(const char *buf, int len, struct cpumask *dstp) { return bitmap_parselist_user(buf, len, ((dstp)->bits), 8); } static inline __attribute__((always_inline)) int cpulist_scnprintf(char *buf, int len, const struct cpumask *srcp) { return bitmap_scnlistprintf(buf, len, ((srcp)->bits), 8); } static inline __attribute__((always_inline)) int cpulist_parse(const char *buf, struct cpumask *dstp) { return bitmap_parselist(buf, ((dstp)->bits), 8); } static inline __attribute__((always_inline)) size_t cpumask_size(void) { return (((8) + (8 * sizeof(long)) - 1) / (8 * sizeof(long))) * sizeof(long); } typedef struct cpumask cpumask_var_t[1]; static inline __attribute__((always_inline)) bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) { return true; } static inline __attribute__((always_inline)) bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node) { return true; } static inline __attribute__((always_inline)) bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags) { cpumask_clear(*mask); return true; } static inline __attribute__((always_inline)) bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node) { cpumask_clear(*mask); return true; } static inline __attribute__((always_inline)) void alloc_bootmem_cpumask_var(cpumask_var_t *mask) { } static inline __attribute__((always_inline)) void free_cpumask_var(cpumask_var_t mask) { } static inline __attribute__((always_inline)) void free_bootmem_cpumask_var(cpumask_var_t mask) { } extern const unsigned long cpu_all_bits[(((8) + (8 * sizeof(long)) - 1) / (8 * sizeof(long)))]; void set_cpu_possible(unsigned int cpu, bool possible); void set_cpu_present(unsigned int cpu, bool present); void set_cpu_online(unsigned int cpu, bool online); void set_cpu_active(unsigned int cpu, bool active); void init_cpu_present(const struct cpumask *src); void init_cpu_possible(const struct cpumask *src); void init_cpu_online(const struct cpumask *src); static inline __attribute__((always_inline)) int __check_is_bitmap(const unsigned long *bitmap) { return 1; } extern const unsigned long cpu_bit_bitmap[32 +1][(((8) + (8 * sizeof(long)) - 1) / (8 * sizeof(long)))]; static inline __attribute__((always_inline)) const struct cpumask *get_cpu_mask(unsigned int cpu) { const unsigned long *p = cpu_bit_bitmap[1 + cpu % 32]; p -= cpu / 32; return ((struct cpumask *)(1 ? (p) : (void *)sizeof(__check_is_bitmap(p)))); } int __first_cpu(const cpumask_t *srcp); int __next_cpu(int n, const cpumask_t *srcp); int __any_online_cpu(const cpumask_t *mask); static inline __attribute__((always_inline)) void __cpu_set(int cpu, volatile cpumask_t *dstp) { set_bit(cpu, dstp->bits); } static inline __attribute__((always_inline)) void __cpu_clear(int cpu, volatile cpumask_t *dstp) { clear_bit(cpu, dstp->bits); } static inline __attribute__((always_inline)) void __cpus_setall(cpumask_t *dstp, int nbits) { bitmap_fill(dstp->bits, nbits); } static inline __attribute__((always_inline)) void __cpus_clear(cpumask_t *dstp, int nbits) { bitmap_zero(dstp->bits, nbits); } static inline __attribute__((always_inline)) int __cpu_test_and_set(int cpu, cpumask_t *addr) { return test_and_set_bit(cpu, addr->bits); } static inline __attribute__((always_inline)) int __cpus_and(cpumask_t *dstp, const cpumask_t *src1p, const cpumask_t *src2p, int nbits) { return bitmap_and(dstp->bits, src1p->bits, src2p->bits, nbits); } static inline __attribute__((always_inline)) void __cpus_or(cpumask_t *dstp, const cpumask_t *src1p, const cpumask_t *src2p, int nbits) { bitmap_or(dstp->bits, src1p->bits, src2p->bits, nbits); } static inline __attribute__((always_inline)) void __cpus_xor(cpumask_t *dstp, const cpumask_t *src1p, const cpumask_t *src2p, int nbits) { bitmap_xor(dstp->bits, src1p->bits, src2p->bits, nbits); } static inline __attribute__((always_inline)) int __cpus_andnot(cpumask_t *dstp, const cpumask_t *src1p, const cpumask_t *src2p, int nbits) { return bitmap_andnot(dstp->bits, src1p->bits, src2p->bits, nbits); } static inline __attribute__((always_inline)) int __cpus_equal(const cpumask_t *src1p, const cpumask_t *src2p, int nbits) { return bitmap_equal(src1p->bits, src2p->bits, nbits); } static inline __attribute__((always_inline)) int __cpus_intersects(const cpumask_t *src1p, const cpumask_t *src2p, int nbits) { return bitmap_intersects(src1p->bits, src2p->bits, nbits); } static inline __attribute__((always_inline)) int __cpus_subset(const cpumask_t *src1p, const cpumask_t *src2p, int nbits) { return bitmap_subset(src1p->bits, src2p->bits, nbits); } static inline __attribute__((always_inline)) int __cpus_empty(const cpumask_t *srcp, int nbits) { return bitmap_empty(srcp->bits, nbits); } static inline __attribute__((always_inline)) int __cpus_weight(const cpumask_t *srcp, int nbits) { return bitmap_weight(srcp->bits, nbits); } static inline __attribute__((always_inline)) void __cpus_shift_left(cpumask_t *dstp, const cpumask_t *srcp, int n, int nbits) { bitmap_shift_left(dstp->bits, srcp->bits, n, nbits); } static inline __attribute__((always_inline)) int paravirt_enabled(void) { return pv_info.paravirt_enabled; } static inline __attribute__((always_inline)) void load_sp0(struct tss_struct *tss, struct thread_struct *thread) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_cpu_ops.load_sp0); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_cpu_ops.load_sp0) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_cpu_ops.load_sp0)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(tss)), "d" ((unsigned long)(thread)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) void __cpuid(unsigned int *eax, unsigned int *ebx, unsigned int *ecx, unsigned int *edx) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_cpu_ops.cpuid); asm volatile("push %[_arg4];" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "lea 4(%%esp),%%esp;" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_cpu_ops.cpuid) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_cpu_ops.cpuid)), [paravirt_clobber] "i" (((1 << 4) - 1)), "0" ((u32)(eax)), "1" ((u32)(ebx)), "2" ((u32)(ecx)), [_arg4] "mr" ((u32)(edx)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) unsigned long paravirt_get_debugreg(int reg) { return ({ unsigned long __ret; unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_cpu_ops.get_debugreg); if (__builtin_constant_p(((sizeof(unsigned long) > sizeof(unsigned long)))) ? !!((sizeof(unsigned long) > sizeof(unsigned long))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/paravirt.h", .line = 39, }; ______r = !!((sizeof(unsigned long) > sizeof(unsigned long))); ______f.miss_hit[______r]++; ______r; })) { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_cpu_ops.get_debugreg) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_cpu_ops.get_debugreg)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(reg)) : "memory", "cc" ); __ret = (unsigned long)((((u64)__edx) << 32) | __eax); } else { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_cpu_ops.get_debugreg) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_cpu_ops.get_debugreg)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(reg)) : "memory", "cc" ); __ret = (unsigned long)__eax; } __ret; }); } static inline __attribute__((always_inline)) void set_debugreg(unsigned long val, int reg) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_cpu_ops.set_debugreg); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_cpu_ops.set_debugreg) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_cpu_ops.set_debugreg)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(reg)), "d" ((unsigned long)(val)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) void clts(void) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_cpu_ops.clts); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_cpu_ops.clts) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_cpu_ops.clts)), [paravirt_clobber] "i" (((1 << 4) - 1)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) unsigned long read_cr0(void) { return ({ unsigned long __ret; unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_cpu_ops.read_cr0); if (__builtin_constant_p(((sizeof(unsigned long) > sizeof(unsigned long)))) ? !!((sizeof(unsigned long) > sizeof(unsigned long))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/paravirt.h", .line = 54, }; ______r = !!((sizeof(unsigned long) > sizeof(unsigned long))); ______f.miss_hit[______r]++; ______r; })) { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_cpu_ops.read_cr0) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_cpu_ops.read_cr0)), [paravirt_clobber] "i" (((1 << 4) - 1)) : "memory", "cc" ); __ret = (unsigned long)((((u64)__edx) << 32) | __eax); } else { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_cpu_ops.read_cr0) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_cpu_ops.read_cr0)), [paravirt_clobber] "i" (((1 << 4) - 1)) : "memory", "cc" ); __ret = (unsigned long)__eax; } __ret; }); } static inline __attribute__((always_inline)) void write_cr0(unsigned long x) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_cpu_ops.write_cr0); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_cpu_ops.write_cr0) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_cpu_ops.write_cr0)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(x)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) unsigned long read_cr2(void) { return ({ unsigned long __ret; unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.read_cr2); if (__builtin_constant_p(((sizeof(unsigned long) > sizeof(unsigned long)))) ? !!((sizeof(unsigned long) > sizeof(unsigned long))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/paravirt.h", .line = 64, }; ______r = !!((sizeof(unsigned long) > sizeof(unsigned long))); ______f.miss_hit[______r]++; ______r; })) { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.read_cr2) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.read_cr2)), [paravirt_clobber] "i" (((1 << 4) - 1)) : "memory", "cc" ); __ret = (unsigned long)((((u64)__edx) << 32) | __eax); } else { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.read_cr2) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.read_cr2)), [paravirt_clobber] "i" (((1 << 4) - 1)) : "memory", "cc" ); __ret = (unsigned long)__eax; } __ret; }); } static inline __attribute__((always_inline)) void write_cr2(unsigned long x) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.write_cr2); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.write_cr2) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.write_cr2)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(x)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) unsigned long read_cr3(void) { return ({ unsigned long __ret; unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.read_cr3); if (__builtin_constant_p(((sizeof(unsigned long) > sizeof(unsigned long)))) ? !!((sizeof(unsigned long) > sizeof(unsigned long))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/paravirt.h", .line = 74, }; ______r = !!((sizeof(unsigned long) > sizeof(unsigned long))); ______f.miss_hit[______r]++; ______r; })) { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.read_cr3) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.read_cr3)), [paravirt_clobber] "i" (((1 << 4) - 1)) : "memory", "cc" ); __ret = (unsigned long)((((u64)__edx) << 32) | __eax); } else { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.read_cr3) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.read_cr3)), [paravirt_clobber] "i" (((1 << 4) - 1)) : "memory", "cc" ); __ret = (unsigned long)__eax; } __ret; }); } static inline __attribute__((always_inline)) void write_cr3(unsigned long x) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.write_cr3); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.write_cr3) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.write_cr3)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(x)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) unsigned long read_cr4(void) { return ({ unsigned long __ret; unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_cpu_ops.read_cr4); if (__builtin_constant_p(((sizeof(unsigned long) > sizeof(unsigned long)))) ? !!((sizeof(unsigned long) > sizeof(unsigned long))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/paravirt.h", .line = 84, }; ______r = !!((sizeof(unsigned long) > sizeof(unsigned long))); ______f.miss_hit[______r]++; ______r; })) { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_cpu_ops.read_cr4) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_cpu_ops.read_cr4)), [paravirt_clobber] "i" (((1 << 4) - 1)) : "memory", "cc" ); __ret = (unsigned long)((((u64)__edx) << 32) | __eax); } else { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_cpu_ops.read_cr4) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_cpu_ops.read_cr4)), [paravirt_clobber] "i" (((1 << 4) - 1)) : "memory", "cc" ); __ret = (unsigned long)__eax; } __ret; }); } static inline __attribute__((always_inline)) unsigned long read_cr4_safe(void) { return ({ unsigned long __ret; unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_cpu_ops.read_cr4_safe); if (__builtin_constant_p(((sizeof(unsigned long) > sizeof(unsigned long)))) ? !!((sizeof(unsigned long) > sizeof(unsigned long))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/paravirt.h", .line = 88, }; ______r = !!((sizeof(unsigned long) > sizeof(unsigned long))); ______f.miss_hit[______r]++; ______r; })) { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_cpu_ops.read_cr4_safe) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_cpu_ops.read_cr4_safe)), [paravirt_clobber] "i" (((1 << 4) - 1)) : "memory", "cc" ); __ret = (unsigned long)((((u64)__edx) << 32) | __eax); } else { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_cpu_ops.read_cr4_safe) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_cpu_ops.read_cr4_safe)), [paravirt_clobber] "i" (((1 << 4) - 1)) : "memory", "cc" ); __ret = (unsigned long)__eax; } __ret; }); } static inline __attribute__((always_inline)) void write_cr4(unsigned long x) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_cpu_ops.write_cr4); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_cpu_ops.write_cr4) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_cpu_ops.write_cr4)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(x)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) void arch_safe_halt(void) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_irq_ops.safe_halt); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_irq_ops.safe_halt) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_irq_ops.safe_halt)), [paravirt_clobber] "i" (((1 << 4) - 1)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) void halt(void) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_irq_ops.halt); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_irq_ops.halt) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_irq_ops.halt)), [paravirt_clobber] "i" (((1 << 4) - 1)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) void wbinvd(void) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_cpu_ops.wbinvd); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_cpu_ops.wbinvd) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_cpu_ops.wbinvd)), [paravirt_clobber] "i" (((1 << 4) - 1)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) u64 paravirt_read_msr(unsigned msr, int *err) { return ({ u64 __ret; unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_cpu_ops.read_msr); if (__builtin_constant_p(((sizeof(u64) > sizeof(unsigned long)))) ? !!((sizeof(u64) > sizeof(unsigned long))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/paravirt.h", .line = 127, }; ______r = !!((sizeof(u64) > sizeof(unsigned long))); ______f.miss_hit[______r]++; ______r; })) { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_cpu_ops.read_msr) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_cpu_ops.read_msr)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(msr)), "d" ((unsigned long)(err)) : "memory", "cc" ); __ret = (u64)((((u64)__edx) << 32) | __eax); } else { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_cpu_ops.read_msr) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_cpu_ops.read_msr)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(msr)), "d" ((unsigned long)(err)) : "memory", "cc" ); __ret = (u64)__eax; } __ret; }); } static inline __attribute__((always_inline)) int paravirt_rdmsr_regs(u32 *regs) { return ({ int __ret; unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_cpu_ops.rdmsr_regs); if (__builtin_constant_p(((sizeof(int) > sizeof(unsigned long)))) ? !!((sizeof(int) > sizeof(unsigned long))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/paravirt.h", .line = 132, }; ______r = !!((sizeof(int) > sizeof(unsigned long))); ______f.miss_hit[______r]++; ______r; })) { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_cpu_ops.rdmsr_regs) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_cpu_ops.rdmsr_regs)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(regs)) : "memory", "cc" ); __ret = (int)((((u64)__edx) << 32) | __eax); } else { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_cpu_ops.rdmsr_regs) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_cpu_ops.rdmsr_regs)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(regs)) : "memory", "cc" ); __ret = (int)__eax; } __ret; }); } static inline __attribute__((always_inline)) int paravirt_write_msr(unsigned msr, unsigned low, unsigned high) { return ({ int __ret; unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_cpu_ops.write_msr); if (__builtin_constant_p(((sizeof(int) > sizeof(unsigned long)))) ? !!((sizeof(int) > sizeof(unsigned long))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/paravirt.h", .line = 137, }; ______r = !!((sizeof(int) > sizeof(unsigned long))); ______f.miss_hit[______r]++; ______r; })) { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_cpu_ops.write_msr) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_cpu_ops.write_msr)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(msr)), "d" ((unsigned long)(low)), "c" ((unsigned long)(high)) : "memory", "cc" ); __ret = (int)((((u64)__edx) << 32) | __eax); } else { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_cpu_ops.write_msr) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_cpu_ops.write_msr)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(msr)), "d" ((unsigned long)(low)), "c" ((unsigned long)(high)) : "memory", "cc" ); __ret = (int)__eax; } __ret; }); } static inline __attribute__((always_inline)) int paravirt_wrmsr_regs(u32 *regs) { return ({ int __ret; unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_cpu_ops.wrmsr_regs); if (__builtin_constant_p(((sizeof(int) > sizeof(unsigned long)))) ? !!((sizeof(int) > sizeof(unsigned long))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/paravirt.h", .line = 142, }; ______r = !!((sizeof(int) > sizeof(unsigned long))); ______f.miss_hit[______r]++; ______r; })) { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_cpu_ops.wrmsr_regs) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_cpu_ops.wrmsr_regs)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(regs)) : "memory", "cc" ); __ret = (int)((((u64)__edx) << 32) | __eax); } else { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_cpu_ops.wrmsr_regs) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_cpu_ops.wrmsr_regs)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(regs)) : "memory", "cc" ); __ret = (int)__eax; } __ret; }); } static inline __attribute__((always_inline)) int rdmsrl_safe(unsigned msr, unsigned long long *p) { int err; *p = paravirt_read_msr(msr, &err); return err; } static inline __attribute__((always_inline)) int rdmsrl_amd_safe(unsigned msr, unsigned long long *p) { u32 gprs[8] = { 0 }; int err; gprs[1] = msr; gprs[7] = 0x9c5a203a; err = paravirt_rdmsr_regs(gprs); *p = gprs[0] | ((u64)gprs[2] << 32); return err; } static inline __attribute__((always_inline)) int wrmsrl_amd_safe(unsigned msr, unsigned long long val) { u32 gprs[8] = { 0 }; gprs[0] = (u32)val; gprs[1] = msr; gprs[2] = val >> 32; gprs[7] = 0x9c5a203a; return paravirt_wrmsr_regs(gprs); } static inline __attribute__((always_inline)) u64 paravirt_read_tsc(void) { return ({ u64 __ret; unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_cpu_ops.read_tsc); if (__builtin_constant_p(((sizeof(u64) > sizeof(unsigned long)))) ? !!((sizeof(u64) > sizeof(unsigned long))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/paravirt.h", .line = 217, }; ______r = !!((sizeof(u64) > sizeof(unsigned long))); ______f.miss_hit[______r]++; ______r; })) { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_cpu_ops.read_tsc) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_cpu_ops.read_tsc)), [paravirt_clobber] "i" (((1 << 4) - 1)) : "memory", "cc" ); __ret = (u64)((((u64)__edx) << 32) | __eax); } else { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_cpu_ops.read_tsc) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_cpu_ops.read_tsc)), [paravirt_clobber] "i" (((1 << 4) - 1)) : "memory", "cc" ); __ret = (u64)__eax; } __ret; }); } static inline __attribute__((always_inline)) unsigned long long paravirt_sched_clock(void) { return ({ unsigned long long __ret; unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_time_ops.sched_clock); if (__builtin_constant_p(((sizeof(unsigned long long) > sizeof(unsigned long)))) ? !!((sizeof(unsigned long long) > sizeof(unsigned long))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/paravirt.h", .line = 230, }; ______r = !!((sizeof(unsigned long long) > sizeof(unsigned long))); ______f.miss_hit[______r]++; ______r; })) { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_time_ops.sched_clock) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_time_ops.sched_clock)), [paravirt_clobber] "i" (((1 << 4) - 1)) : "memory", "cc" ); __ret = (unsigned long long)((((u64)__edx) << 32) | __eax); } else { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_time_ops.sched_clock) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_time_ops.sched_clock)), [paravirt_clobber] "i" (((1 << 4) - 1)) : "memory", "cc" ); __ret = (unsigned long long)__eax; } __ret; }); } static inline __attribute__((always_inline)) unsigned long long paravirt_read_pmc(int counter) { return ({ u64 __ret; unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_cpu_ops.read_pmc); if (__builtin_constant_p(((sizeof(u64) > sizeof(unsigned long)))) ? !!((sizeof(u64) > sizeof(unsigned long))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/paravirt.h", .line = 235, }; ______r = !!((sizeof(u64) > sizeof(unsigned long))); ______f.miss_hit[______r]++; ______r; })) { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_cpu_ops.read_pmc) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_cpu_ops.read_pmc)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(counter)) : "memory", "cc" ); __ret = (u64)((((u64)__edx) << 32) | __eax); } else { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_cpu_ops.read_pmc) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_cpu_ops.read_pmc)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(counter)) : "memory", "cc" ); __ret = (u64)__eax; } __ret; }); } static inline __attribute__((always_inline)) unsigned long long paravirt_rdtscp(unsigned int *aux) { return ({ u64 __ret; unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_cpu_ops.read_tscp); if (__builtin_constant_p(((sizeof(u64) > sizeof(unsigned long)))) ? !!((sizeof(u64) > sizeof(unsigned long))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/paravirt.h", .line = 247, }; ______r = !!((sizeof(u64) > sizeof(unsigned long))); ______f.miss_hit[______r]++; ______r; })) { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_cpu_ops.read_tscp) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_cpu_ops.read_tscp)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(aux)) : "memory", "cc" ); __ret = (u64)((((u64)__edx) << 32) | __eax); } else { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_cpu_ops.read_tscp) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_cpu_ops.read_tscp)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(aux)) : "memory", "cc" ); __ret = (u64)__eax; } __ret; }); } static inline __attribute__((always_inline)) void paravirt_alloc_ldt(struct desc_struct *ldt, unsigned entries) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_cpu_ops.alloc_ldt); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_cpu_ops.alloc_ldt) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_cpu_ops.alloc_ldt)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(ldt)), "d" ((unsigned long)(entries)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) void paravirt_free_ldt(struct desc_struct *ldt, unsigned entries) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_cpu_ops.free_ldt); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_cpu_ops.free_ldt) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_cpu_ops.free_ldt)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(ldt)), "d" ((unsigned long)(entries)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) void load_TR_desc(void) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_cpu_ops.load_tr_desc); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_cpu_ops.load_tr_desc) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_cpu_ops.load_tr_desc)), [paravirt_clobber] "i" (((1 << 4) - 1)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) void load_gdt(const struct desc_ptr *dtr) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_cpu_ops.load_gdt); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_cpu_ops.load_gdt) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_cpu_ops.load_gdt)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(dtr)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) void load_idt(const struct desc_ptr *dtr) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_cpu_ops.load_idt); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_cpu_ops.load_idt) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_cpu_ops.load_idt)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(dtr)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) void set_ldt(const void *addr, unsigned entries) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_cpu_ops.set_ldt); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_cpu_ops.set_ldt) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_cpu_ops.set_ldt)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(addr)), "d" ((unsigned long)(entries)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) void store_gdt(struct desc_ptr *dtr) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_cpu_ops.store_gdt); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_cpu_ops.store_gdt) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_cpu_ops.store_gdt)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(dtr)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) void store_idt(struct desc_ptr *dtr) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_cpu_ops.store_idt); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_cpu_ops.store_idt) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_cpu_ops.store_idt)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(dtr)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) unsigned long paravirt_store_tr(void) { return ({ unsigned long __ret; unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_cpu_ops.store_tr); if (__builtin_constant_p(((sizeof(unsigned long) > sizeof(unsigned long)))) ? !!((sizeof(unsigned long) > sizeof(unsigned long))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/paravirt.h", .line = 302, }; ______r = !!((sizeof(unsigned long) > sizeof(unsigned long))); ______f.miss_hit[______r]++; ______r; })) { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_cpu_ops.store_tr) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_cpu_ops.store_tr)), [paravirt_clobber] "i" (((1 << 4) - 1)) : "memory", "cc" ); __ret = (unsigned long)((((u64)__edx) << 32) | __eax); } else { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_cpu_ops.store_tr) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_cpu_ops.store_tr)), [paravirt_clobber] "i" (((1 << 4) - 1)) : "memory", "cc" ); __ret = (unsigned long)__eax; } __ret; }); } static inline __attribute__((always_inline)) void load_TLS(struct thread_struct *t, unsigned cpu) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_cpu_ops.load_tls); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_cpu_ops.load_tls) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_cpu_ops.load_tls)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(t)), "d" ((unsigned long)(cpu)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) void write_ldt_entry(struct desc_struct *dt, int entry, const void *desc) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_cpu_ops.write_ldt_entry); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_cpu_ops.write_ldt_entry) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_cpu_ops.write_ldt_entry)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(dt)), "d" ((unsigned long)(entry)), "c" ((unsigned long)(desc)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) void write_gdt_entry(struct desc_struct *dt, int entry, void *desc, int type) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_cpu_ops.write_gdt_entry); asm volatile("push %[_arg4];" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "lea 4(%%esp),%%esp;" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_cpu_ops.write_gdt_entry) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_cpu_ops.write_gdt_entry)), [paravirt_clobber] "i" (((1 << 4) - 1)), "0" ((u32)(dt)), "1" ((u32)(entry)), "2" ((u32)(desc)), [_arg4] "mr" ((u32)(type)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) void write_idt_entry(gate_desc *dt, int entry, const gate_desc *g) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_cpu_ops.write_idt_entry); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_cpu_ops.write_idt_entry) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_cpu_ops.write_idt_entry)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(dt)), "d" ((unsigned long)(entry)), "c" ((unsigned long)(g)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) void set_iopl_mask(unsigned mask) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_cpu_ops.set_iopl_mask); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_cpu_ops.set_iopl_mask) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_cpu_ops.set_iopl_mask)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(mask)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) void slow_down_io(void) { pv_cpu_ops.io_delay(); } static inline __attribute__((always_inline)) void startup_ipi_hook(int phys_apicid, unsigned long start_eip, unsigned long start_esp) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_apic_ops.startup_ipi_hook); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_apic_ops.startup_ipi_hook) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_apic_ops.startup_ipi_hook)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(phys_apicid)), "d" ((unsigned long)(start_eip)), "c" ((unsigned long)(start_esp)) : "memory", "cc" ); }) ; } static inline __attribute__((always_inline)) void paravirt_activate_mm(struct mm_struct *prev, struct mm_struct *next) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.activate_mm); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.activate_mm) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.activate_mm)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(prev)), "d" ((unsigned long)(next)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) void arch_dup_mmap(struct mm_struct *oldmm, struct mm_struct *mm) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.dup_mmap); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.dup_mmap) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.dup_mmap)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(oldmm)), "d" ((unsigned long)(mm)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) void arch_exit_mmap(struct mm_struct *mm) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.exit_mmap); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.exit_mmap) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.exit_mmap)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(mm)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) void __flush_tlb(void) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.flush_tlb_user); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.flush_tlb_user) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.flush_tlb_user)), [paravirt_clobber] "i" (((1 << 4) - 1)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) void __flush_tlb_global(void) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.flush_tlb_kernel); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.flush_tlb_kernel) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.flush_tlb_kernel)), [paravirt_clobber] "i" (((1 << 4) - 1)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) void __flush_tlb_single(unsigned long addr) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.flush_tlb_single); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.flush_tlb_single) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.flush_tlb_single)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(addr)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) void flush_tlb_others(const struct cpumask *cpumask, struct mm_struct *mm, unsigned long va) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.flush_tlb_others); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.flush_tlb_others) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.flush_tlb_others)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(cpumask)), "d" ((unsigned long)(mm)), "c" ((unsigned long)(va)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) int paravirt_pgd_alloc(struct mm_struct *mm) { return ({ int __ret; unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.pgd_alloc); if (__builtin_constant_p(((sizeof(int) > sizeof(unsigned long)))) ? !!((sizeof(int) > sizeof(unsigned long))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/paravirt.h", .line = 397, }; ______r = !!((sizeof(int) > sizeof(unsigned long))); ______f.miss_hit[______r]++; ______r; })) { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.pgd_alloc) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.pgd_alloc)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(mm)) : "memory", "cc" ); __ret = (int)((((u64)__edx) << 32) | __eax); } else { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.pgd_alloc) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.pgd_alloc)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(mm)) : "memory", "cc" ); __ret = (int)__eax; } __ret; }); } static inline __attribute__((always_inline)) void paravirt_pgd_free(struct mm_struct *mm, pgd_t *pgd) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.pgd_free); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.pgd_free) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.pgd_free)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(mm)), "d" ((unsigned long)(pgd)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) void paravirt_alloc_pte(struct mm_struct *mm, unsigned long pfn) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.alloc_pte); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.alloc_pte) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.alloc_pte)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(mm)), "d" ((unsigned long)(pfn)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) void paravirt_release_pte(unsigned long pfn) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.release_pte); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.release_pte) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.release_pte)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(pfn)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) void paravirt_alloc_pmd(struct mm_struct *mm, unsigned long pfn) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.alloc_pmd); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.alloc_pmd) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.alloc_pmd)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(mm)), "d" ((unsigned long)(pfn)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) void paravirt_release_pmd(unsigned long pfn) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.release_pmd); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.release_pmd) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.release_pmd)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(pfn)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) void paravirt_alloc_pud(struct mm_struct *mm, unsigned long pfn) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.alloc_pud); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.alloc_pud) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.alloc_pud)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(mm)), "d" ((unsigned long)(pfn)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) void paravirt_release_pud(unsigned long pfn) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.release_pud); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.release_pud) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.release_pud)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(pfn)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) void pte_update(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.pte_update); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.pte_update) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.pte_update)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(mm)), "d" ((unsigned long)(addr)), "c" ((unsigned long)(ptep)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) void pmd_update(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.pmd_update); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.pmd_update) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.pmd_update)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(mm)), "d" ((unsigned long)(addr)), "c" ((unsigned long)(pmdp)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) void pte_update_defer(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.pte_update_defer); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.pte_update_defer) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.pte_update_defer)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(mm)), "d" ((unsigned long)(addr)), "c" ((unsigned long)(ptep)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) void pmd_update_defer(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.pmd_update_defer); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.pmd_update_defer) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.pmd_update_defer)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(mm)), "d" ((unsigned long)(addr)), "c" ((unsigned long)(pmdp)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) pte_t __pte(pteval_t val) { pteval_t ret; if (__builtin_constant_p(((sizeof(pteval_t) > sizeof(long)))) ? !!((sizeof(pteval_t) > sizeof(long))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/paravirt.h", .line = 460, }; ______r = !!((sizeof(pteval_t) > sizeof(long))); ______f.miss_hit[______r]++; ______r; })) ret = ({ pteval_t __ret; unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.make_pte.func); if (__builtin_constant_p(((sizeof(pteval_t) > sizeof(unsigned long)))) ? !!((sizeof(pteval_t) > sizeof(unsigned long))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/paravirt.h" , .line = 463 , }; ______r = !!((sizeof(pteval_t) > sizeof(unsigned long))); ______f.miss_hit[______r]++; ______r; })) { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.make_pte.func) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.make_pte.func)), [paravirt_clobber] "i" (((1 << 0) | (1 << 2))), "a" ((unsigned long)(val)), "d" ((unsigned long)((u64)val >> 32)) : "memory", "cc" ); __ret = (pteval_t)((((u64)__edx) << 32) | __eax); } else { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.make_pte.func) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.make_pte.func)), [paravirt_clobber] "i" (((1 << 0) | (1 << 2))), "a" ((unsigned long)(val)), "d" ((unsigned long)((u64)val >> 32)) : "memory", "cc" ); __ret = (pteval_t)__eax; } __ret; }) ; else ret = ({ pteval_t __ret; unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.make_pte.func); if (__builtin_constant_p(((sizeof(pteval_t) > sizeof(unsigned long)))) ? !!((sizeof(pteval_t) > sizeof(unsigned long))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/paravirt.h" , .line = 467 , }; ______r = !!((sizeof(pteval_t) > sizeof(unsigned long))); ______f.miss_hit[______r]++; ______r; })) { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.make_pte.func) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.make_pte.func)), [paravirt_clobber] "i" (((1 << 0) | (1 << 2))), "a" ((unsigned long)(val)) : "memory", "cc" ); __ret = (pteval_t)((((u64)__edx) << 32) | __eax); } else { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.make_pte.func) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.make_pte.func)), [paravirt_clobber] "i" (((1 << 0) | (1 << 2))), "a" ((unsigned long)(val)) : "memory", "cc" ); __ret = (pteval_t)__eax; } __ret; }) ; return (pte_t) { .pte = ret }; } static inline __attribute__((always_inline)) pteval_t pte_val(pte_t pte) { pteval_t ret; if (__builtin_constant_p(((sizeof(pteval_t) > sizeof(long)))) ? !!((sizeof(pteval_t) > sizeof(long))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/paravirt.h", .line = 476, }; ______r = !!((sizeof(pteval_t) > sizeof(long))); ______f.miss_hit[______r]++; ______r; })) ret = ({ pteval_t __ret; unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.pte_val.func); if (__builtin_constant_p(((sizeof(pteval_t) > sizeof(unsigned long)))) ? !!((sizeof(pteval_t) > sizeof(unsigned long))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/paravirt.h" , .line = 478 , }; ______r = !!((sizeof(pteval_t) > sizeof(unsigned long))); ______f.miss_hit[______r]++; ______r; })) { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.pte_val.func) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.pte_val.func)), [paravirt_clobber] "i" (((1 << 0) | (1 << 2))), "a" ((unsigned long)(pte.pte)), "d" ((unsigned long)((u64)pte.pte >> 32)) : "memory", "cc" ); __ret = (pteval_t)((((u64)__edx) << 32) | __eax); } else { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.pte_val.func) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.pte_val.func)), [paravirt_clobber] "i" (((1 << 0) | (1 << 2))), "a" ((unsigned long)(pte.pte)), "d" ((unsigned long)((u64)pte.pte >> 32)) : "memory", "cc" ); __ret = (pteval_t)__eax; } __ret; }) ; else ret = ({ pteval_t __ret; unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.pte_val.func); if (__builtin_constant_p(((sizeof(pteval_t) > sizeof(unsigned long)))) ? !!((sizeof(pteval_t) > sizeof(unsigned long))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/paravirt.h" , .line = 481 , }; ______r = !!((sizeof(pteval_t) > sizeof(unsigned long))); ______f.miss_hit[______r]++; ______r; })) { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.pte_val.func) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.pte_val.func)), [paravirt_clobber] "i" (((1 << 0) | (1 << 2))), "a" ((unsigned long)(pte.pte)) : "memory", "cc" ); __ret = (pteval_t)((((u64)__edx) << 32) | __eax); } else { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.pte_val.func) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.pte_val.func)), [paravirt_clobber] "i" (((1 << 0) | (1 << 2))), "a" ((unsigned long)(pte.pte)) : "memory", "cc" ); __ret = (pteval_t)__eax; } __ret; }) ; return ret; } static inline __attribute__((always_inline)) pgd_t __pgd(pgdval_t val) { pgdval_t ret; if (__builtin_constant_p(((sizeof(pgdval_t) > sizeof(long)))) ? !!((sizeof(pgdval_t) > sizeof(long))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/paravirt.h", .line = 490, }; ______r = !!((sizeof(pgdval_t) > sizeof(long))); ______f.miss_hit[______r]++; ______r; })) ret = ({ pgdval_t __ret; unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.make_pgd.func); if (__builtin_constant_p(((sizeof(pgdval_t) > sizeof(unsigned long)))) ? !!((sizeof(pgdval_t) > sizeof(unsigned long))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/paravirt.h" , .line = 492 , }; ______r = !!((sizeof(pgdval_t) > sizeof(unsigned long))); ______f.miss_hit[______r]++; ______r; })) { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.make_pgd.func) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.make_pgd.func)), [paravirt_clobber] "i" (((1 << 0) | (1 << 2))), "a" ((unsigned long)(val)), "d" ((unsigned long)((u64)val >> 32)) : "memory", "cc" ); __ret = (pgdval_t)((((u64)__edx) << 32) | __eax); } else { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.make_pgd.func) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.make_pgd.func)), [paravirt_clobber] "i" (((1 << 0) | (1 << 2))), "a" ((unsigned long)(val)), "d" ((unsigned long)((u64)val >> 32)) : "memory", "cc" ); __ret = (pgdval_t)__eax; } __ret; }) ; else ret = ({ pgdval_t __ret; unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.make_pgd.func); if (__builtin_constant_p(((sizeof(pgdval_t) > sizeof(unsigned long)))) ? !!((sizeof(pgdval_t) > sizeof(unsigned long))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/paravirt.h" , .line = 495 , }; ______r = !!((sizeof(pgdval_t) > sizeof(unsigned long))); ______f.miss_hit[______r]++; ______r; })) { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.make_pgd.func) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.make_pgd.func)), [paravirt_clobber] "i" (((1 << 0) | (1 << 2))), "a" ((unsigned long)(val)) : "memory", "cc" ); __ret = (pgdval_t)((((u64)__edx) << 32) | __eax); } else { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.make_pgd.func) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.make_pgd.func)), [paravirt_clobber] "i" (((1 << 0) | (1 << 2))), "a" ((unsigned long)(val)) : "memory", "cc" ); __ret = (pgdval_t)__eax; } __ret; }) ; return (pgd_t) { ret }; } static inline __attribute__((always_inline)) pgdval_t pgd_val(pgd_t pgd) { pgdval_t ret; if (__builtin_constant_p(((sizeof(pgdval_t) > sizeof(long)))) ? !!((sizeof(pgdval_t) > sizeof(long))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/paravirt.h", .line = 504, }; ______r = !!((sizeof(pgdval_t) > sizeof(long))); ______f.miss_hit[______r]++; ______r; })) ret = ({ pgdval_t __ret; unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.pgd_val.func); if (__builtin_constant_p(((sizeof(pgdval_t) > sizeof(unsigned long)))) ? !!((sizeof(pgdval_t) > sizeof(unsigned long))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/paravirt.h" , .line = 506 , }; ______r = !!((sizeof(pgdval_t) > sizeof(unsigned long))); ______f.miss_hit[______r]++; ______r; })) { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.pgd_val.func) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.pgd_val.func)), [paravirt_clobber] "i" (((1 << 0) | (1 << 2))), "a" ((unsigned long)(pgd.pgd)), "d" ((unsigned long)((u64)pgd.pgd >> 32)) : "memory", "cc" ); __ret = (pgdval_t)((((u64)__edx) << 32) | __eax); } else { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.pgd_val.func) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.pgd_val.func)), [paravirt_clobber] "i" (((1 << 0) | (1 << 2))), "a" ((unsigned long)(pgd.pgd)), "d" ((unsigned long)((u64)pgd.pgd >> 32)) : "memory", "cc" ); __ret = (pgdval_t)__eax; } __ret; }) ; else ret = ({ pgdval_t __ret; unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.pgd_val.func); if (__builtin_constant_p(((sizeof(pgdval_t) > sizeof(unsigned long)))) ? !!((sizeof(pgdval_t) > sizeof(unsigned long))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/paravirt.h" , .line = 509 , }; ______r = !!((sizeof(pgdval_t) > sizeof(unsigned long))); ______f.miss_hit[______r]++; ______r; })) { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.pgd_val.func) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.pgd_val.func)), [paravirt_clobber] "i" (((1 << 0) | (1 << 2))), "a" ((unsigned long)(pgd.pgd)) : "memory", "cc" ); __ret = (pgdval_t)((((u64)__edx) << 32) | __eax); } else { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.pgd_val.func) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.pgd_val.func)), [paravirt_clobber] "i" (((1 << 0) | (1 << 2))), "a" ((unsigned long)(pgd.pgd)) : "memory", "cc" ); __ret = (pgdval_t)__eax; } __ret; }) ; return ret; } static inline __attribute__((always_inline)) pte_t ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { pteval_t ret; ret = ({ pteval_t __ret; unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.ptep_modify_prot_start); if (__builtin_constant_p(((sizeof(pteval_t) > sizeof(unsigned long)))) ? !!((sizeof(pteval_t) > sizeof(unsigned long))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/paravirt.h" , .line = 521 , }; ______r = !!((sizeof(pteval_t) > sizeof(unsigned long))); ______f.miss_hit[______r]++; ______r; })) { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.ptep_modify_prot_start) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.ptep_modify_prot_start)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(mm)), "d" ((unsigned long)(addr)), "c" ((unsigned long)(ptep)) : "memory", "cc" ); __ret = (pteval_t)((((u64)__edx) << 32) | __eax); } else { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.ptep_modify_prot_start) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.ptep_modify_prot_start)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(mm)), "d" ((unsigned long)(addr)), "c" ((unsigned long)(ptep)) : "memory", "cc" ); __ret = (pteval_t)__eax; } __ret; }) ; return (pte_t) { .pte = ret }; } static inline __attribute__((always_inline)) void ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte) { if (__builtin_constant_p(((sizeof(pteval_t) > sizeof(long)))) ? !!((sizeof(pteval_t) > sizeof(long))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/paravirt.h", .line = 529, }; ______r = !!((sizeof(pteval_t) > sizeof(long))); ______f.miss_hit[______r]++; ______r; })) pv_mmu_ops.ptep_modify_prot_commit(mm, addr, ptep, pte); else ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.ptep_modify_prot_commit); asm volatile("push %[_arg4];" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "lea 4(%%esp),%%esp;" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.ptep_modify_prot_commit) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.ptep_modify_prot_commit)), [paravirt_clobber] "i" (((1 << 4) - 1)), "0" ((u32)(mm)), "1" ((u32)(addr)), "2" ((u32)(ptep)), [_arg4] "mr" ((u32)(pte.pte)) : "memory", "cc" ); }) ; } static inline __attribute__((always_inline)) void set_pte(pte_t *ptep, pte_t pte) { if (__builtin_constant_p(((sizeof(pteval_t) > sizeof(long)))) ? !!((sizeof(pteval_t) > sizeof(long))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/paravirt.h", .line = 539, }; ______r = !!((sizeof(pteval_t) > sizeof(long))); ______f.miss_hit[______r]++; ______r; })) ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.set_pte); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.set_pte) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.set_pte)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(ptep)), "d" ((unsigned long)(pte.pte)), "c" ((unsigned long)((u64)pte.pte >> 32)) : "memory", "cc" ); }) ; else ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.set_pte); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.set_pte) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.set_pte)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(ptep)), "d" ((unsigned long)(pte.pte)) : "memory", "cc" ); }) ; } static inline __attribute__((always_inline)) void set_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte) { if (__builtin_constant_p(((sizeof(pteval_t) > sizeof(long)))) ? !!((sizeof(pteval_t) > sizeof(long))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/paravirt.h", .line = 550, }; ______r = !!((sizeof(pteval_t) > sizeof(long))); ______f.miss_hit[______r]++; ______r; })) pv_mmu_ops.set_pte_at(mm, addr, ptep, pte); else ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.set_pte_at); asm volatile("push %[_arg4];" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "lea 4(%%esp),%%esp;" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.set_pte_at) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.set_pte_at)), [paravirt_clobber] "i" (((1 << 4) - 1)), "0" ((u32)(mm)), "1" ((u32)(addr)), "2" ((u32)(ptep)), [_arg4] "mr" ((u32)(pte.pte)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) void set_pmd_at(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp, pmd_t pmd) { if (__builtin_constant_p(((sizeof(pmdval_t) > sizeof(long)))) ? !!((sizeof(pmdval_t) > sizeof(long))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/paravirt.h", .line = 561, }; ______r = !!((sizeof(pmdval_t) > sizeof(long))); ______f.miss_hit[______r]++; ______r; })) pv_mmu_ops.set_pmd_at(mm, addr, pmdp, pmd); else ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.set_pmd_at); asm volatile("push %[_arg4];" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "lea 4(%%esp),%%esp;" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.set_pmd_at) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.set_pmd_at)), [paravirt_clobber] "i" (((1 << 4) - 1)), "0" ((u32)(mm)), "1" ((u32)(addr)), "2" ((u32)(pmdp)), [_arg4] "mr" ((u32)(native_pmd_val(pmd))) : "memory", "cc" ); }) ; } static inline __attribute__((always_inline)) void set_pmd(pmd_t *pmdp, pmd_t pmd) { pmdval_t val = native_pmd_val(pmd); if (__builtin_constant_p(((sizeof(pmdval_t) > sizeof(long)))) ? !!((sizeof(pmdval_t) > sizeof(long))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/paravirt.h", .line = 574, }; ______r = !!((sizeof(pmdval_t) > sizeof(long))); ______f.miss_hit[______r]++; ______r; })) ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.set_pmd); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.set_pmd) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.set_pmd)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(pmdp)), "d" ((unsigned long)(val)), "c" ((unsigned long)((u64)val >> 32)) : "memory", "cc" ); }); else ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.set_pmd); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.set_pmd) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.set_pmd)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(pmdp)), "d" ((unsigned long)(val)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) pmd_t __pmd(pmdval_t val) { pmdval_t ret; if (__builtin_constant_p(((sizeof(pmdval_t) > sizeof(long)))) ? !!((sizeof(pmdval_t) > sizeof(long))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/paravirt.h", .line = 585, }; ______r = !!((sizeof(pmdval_t) > sizeof(long))); ______f.miss_hit[______r]++; ______r; })) ret = ({ pmdval_t __ret; unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.make_pmd.func); if (__builtin_constant_p(((sizeof(pmdval_t) > sizeof(unsigned long)))) ? !!((sizeof(pmdval_t) > sizeof(unsigned long))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/paravirt.h" , .line = 587 , }; ______r = !!((sizeof(pmdval_t) > sizeof(unsigned long))); ______f.miss_hit[______r]++; ______r; })) { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.make_pmd.func) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.make_pmd.func)), [paravirt_clobber] "i" (((1 << 0) | (1 << 2))), "a" ((unsigned long)(val)), "d" ((unsigned long)((u64)val >> 32)) : "memory", "cc" ); __ret = (pmdval_t)((((u64)__edx) << 32) | __eax); } else { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.make_pmd.func) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.make_pmd.func)), [paravirt_clobber] "i" (((1 << 0) | (1 << 2))), "a" ((unsigned long)(val)), "d" ((unsigned long)((u64)val >> 32)) : "memory", "cc" ); __ret = (pmdval_t)__eax; } __ret; }) ; else ret = ({ pmdval_t __ret; unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.make_pmd.func); if (__builtin_constant_p(((sizeof(pmdval_t) > sizeof(unsigned long)))) ? !!((sizeof(pmdval_t) > sizeof(unsigned long))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/paravirt.h" , .line = 590 , }; ______r = !!((sizeof(pmdval_t) > sizeof(unsigned long))); ______f.miss_hit[______r]++; ______r; })) { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.make_pmd.func) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.make_pmd.func)), [paravirt_clobber] "i" (((1 << 0) | (1 << 2))), "a" ((unsigned long)(val)) : "memory", "cc" ); __ret = (pmdval_t)((((u64)__edx) << 32) | __eax); } else { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.make_pmd.func) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.make_pmd.func)), [paravirt_clobber] "i" (((1 << 0) | (1 << 2))), "a" ((unsigned long)(val)) : "memory", "cc" ); __ret = (pmdval_t)__eax; } __ret; }) ; return (pmd_t) { ret }; } static inline __attribute__((always_inline)) pmdval_t pmd_val(pmd_t pmd) { pmdval_t ret; if (__builtin_constant_p(((sizeof(pmdval_t) > sizeof(long)))) ? !!((sizeof(pmdval_t) > sizeof(long))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/paravirt.h", .line = 599, }; ______r = !!((sizeof(pmdval_t) > sizeof(long))); ______f.miss_hit[______r]++; ______r; })) ret = ({ pmdval_t __ret; unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.pmd_val.func); if (__builtin_constant_p(((sizeof(pmdval_t) > sizeof(unsigned long)))) ? !!((sizeof(pmdval_t) > sizeof(unsigned long))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/paravirt.h" , .line = 601 , }; ______r = !!((sizeof(pmdval_t) > sizeof(unsigned long))); ______f.miss_hit[______r]++; ______r; })) { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.pmd_val.func) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.pmd_val.func)), [paravirt_clobber] "i" (((1 << 0) | (1 << 2))), "a" ((unsigned long)(pmd.pmd)), "d" ((unsigned long)((u64)pmd.pmd >> 32)) : "memory", "cc" ); __ret = (pmdval_t)((((u64)__edx) << 32) | __eax); } else { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.pmd_val.func) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.pmd_val.func)), [paravirt_clobber] "i" (((1 << 0) | (1 << 2))), "a" ((unsigned long)(pmd.pmd)), "d" ((unsigned long)((u64)pmd.pmd >> 32)) : "memory", "cc" ); __ret = (pmdval_t)__eax; } __ret; }) ; else ret = ({ pmdval_t __ret; unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.pmd_val.func); if (__builtin_constant_p(((sizeof(pmdval_t) > sizeof(unsigned long)))) ? !!((sizeof(pmdval_t) > sizeof(unsigned long))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/paravirt.h" , .line = 604 , }; ______r = !!((sizeof(pmdval_t) > sizeof(unsigned long))); ______f.miss_hit[______r]++; ______r; })) { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.pmd_val.func) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.pmd_val.func)), [paravirt_clobber] "i" (((1 << 0) | (1 << 2))), "a" ((unsigned long)(pmd.pmd)) : "memory", "cc" ); __ret = (pmdval_t)((((u64)__edx) << 32) | __eax); } else { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.pmd_val.func) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.pmd_val.func)), [paravirt_clobber] "i" (((1 << 0) | (1 << 2))), "a" ((unsigned long)(pmd.pmd)) : "memory", "cc" ); __ret = (pmdval_t)__eax; } __ret; }) ; return ret; } static inline __attribute__((always_inline)) void set_pud(pud_t *pudp, pud_t pud) { pudval_t val = native_pud_val(pud); if (__builtin_constant_p(((sizeof(pudval_t) > sizeof(long)))) ? !!((sizeof(pudval_t) > sizeof(long))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/paravirt.h", .line = 613, }; ______r = !!((sizeof(pudval_t) > sizeof(long))); ______f.miss_hit[______r]++; ______r; })) ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.set_pud); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.set_pud) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.set_pud)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(pudp)), "d" ((unsigned long)(val)), "c" ((unsigned long)((u64)val >> 32)) : "memory", "cc" ); }) ; else ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.set_pud); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.set_pud) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.set_pud)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(pudp)), "d" ((unsigned long)(val)) : "memory", "cc" ); }) ; } static inline __attribute__((always_inline)) void set_pte_atomic(pte_t *ptep, pte_t pte) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.set_pte_atomic); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.set_pte_atomic) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.set_pte_atomic)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(ptep)), "d" ((unsigned long)(pte.pte)), "c" ((unsigned long)(pte.pte >> 32)) : "memory", "cc" ); }) ; } static inline __attribute__((always_inline)) void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.pte_clear); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.pte_clear) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.pte_clear)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(mm)), "d" ((unsigned long)(addr)), "c" ((unsigned long)(ptep)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) void pmd_clear(pmd_t *pmdp) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.pmd_clear); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.pmd_clear) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.pmd_clear)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(pmdp)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) void arch_start_context_switch(struct task_struct *prev) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_cpu_ops.start_context_switch); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_cpu_ops.start_context_switch) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_cpu_ops.start_context_switch)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(prev)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) void arch_end_context_switch(struct task_struct *next) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_cpu_ops.end_context_switch); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_cpu_ops.end_context_switch) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_cpu_ops.end_context_switch)), [paravirt_clobber] "i" (((1 << 4) - 1)), "a" ((unsigned long)(next)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) void arch_enter_lazy_mmu_mode(void) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.lazy_mode.enter); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.lazy_mode.enter) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.lazy_mode.enter)), [paravirt_clobber] "i" (((1 << 4) - 1)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) void arch_leave_lazy_mmu_mode(void) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_mmu_ops.lazy_mode.leave); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx), "=c" (__ecx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_mmu_ops.lazy_mode.leave) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_mmu_ops.lazy_mode.leave)), [paravirt_clobber] "i" (((1 << 4) - 1)) : "memory", "cc" ); }); } void arch_flush_lazy_mmu_mode(void); static inline __attribute__((always_inline)) void __set_fixmap(unsigned idx, phys_addr_t phys, pgprot_t flags) { pv_mmu_ops.set_fixmap(idx, phys, flags); } static inline __attribute__((always_inline)) __attribute__((no_instrument_function)) unsigned long arch_local_save_flags(void) { return ({ unsigned long __ret; unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_irq_ops.save_fl.func); if (__builtin_constant_p(((sizeof(unsigned long) > sizeof(unsigned long)))) ? !!((sizeof(unsigned long) > sizeof(unsigned long))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/paravirt.h", .line = 853, }; ______r = !!((sizeof(unsigned long) > sizeof(unsigned long))); ______f.miss_hit[______r]++; ______r; })) { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_irq_ops.save_fl.func) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_irq_ops.save_fl.func)), [paravirt_clobber] "i" (((1 << 0) | (1 << 2))) : "memory", "cc" ); __ret = (unsigned long)((((u64)__edx) << 32) | __eax); } else { asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_irq_ops.save_fl.func) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_irq_ops.save_fl.func)), [paravirt_clobber] "i" (((1 << 0) | (1 << 2))) : "memory", "cc" ); __ret = (unsigned long)__eax; } __ret; }); } static inline __attribute__((always_inline)) __attribute__((no_instrument_function)) void arch_local_irq_restore(unsigned long f) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_irq_ops.restore_fl.func); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_irq_ops.restore_fl.func) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_irq_ops.restore_fl.func)), [paravirt_clobber] "i" (((1 << 0) | (1 << 2))), "a" ((unsigned long)(f)) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) __attribute__((no_instrument_function)) void arch_local_irq_disable(void) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_irq_ops.irq_disable.func); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_irq_ops.irq_disable.func) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_irq_ops.irq_disable.func)), [paravirt_clobber] "i" (((1 << 0) | (1 << 2))) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) __attribute__((no_instrument_function)) void arch_local_irq_enable(void) { ({ unsigned long __eax = __eax, __edx = __edx, __ecx = __ecx; ((void)pv_irq_ops.irq_enable.func); asm volatile("" "771:\n\t" "call *%c[paravirt_opptr];" "\n" "772:\n" ".pushsection .parainstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " " 771b\n" " .byte " "%c[paravirt_typenum]" "\n" " .byte 772b-771b\n" " .short " "%c[paravirt_clobber]" "\n" ".popsection\n" "" : "=a" (__eax), "=d" (__edx) : [paravirt_typenum] "i" ((__builtin_offsetof(struct paravirt_patch_template,pv_irq_ops.irq_enable.func) / sizeof(void *))), [paravirt_opptr] "i" (&(pv_irq_ops.irq_enable.func)), [paravirt_clobber] "i" (((1 << 0) | (1 << 2))) : "memory", "cc" ); }); } static inline __attribute__((always_inline)) __attribute__((no_instrument_function)) unsigned long arch_local_irq_save(void) { unsigned long f; f = arch_local_save_flags(); arch_local_irq_disable(); return f; } extern void default_banner(void); static inline __attribute__((always_inline)) int arch_irqs_disabled_flags(unsigned long flags) { return !(flags & 0x00000200); } static inline __attribute__((always_inline)) int arch_irqs_disabled(void) { unsigned long flags = arch_local_save_flags(); return arch_irqs_disabled_flags(flags); } extern void trace_softirqs_on(unsigned long ip); extern void trace_softirqs_off(unsigned long ip); extern void trace_hardirqs_on(void); extern void trace_hardirqs_off(void); extern void stop_critical_timings(void); extern void start_critical_timings(void); struct task_struct; struct task_struct *__switch_to(struct task_struct *prev, struct task_struct *next); struct tss_struct; void __switch_to_xtra(struct task_struct *prev_p, struct task_struct *next_p, struct tss_struct *tss); extern void show_regs_common(void); extern void native_load_gs_index(unsigned); static inline __attribute__((always_inline)) unsigned long get_limit(unsigned long segment) { unsigned long __limit; asm("lsll %1,%0" : "=r" (__limit) : "r" (segment)); return __limit + 1; } static inline __attribute__((always_inline)) void native_clts(void) { asm volatile("clts"); } static unsigned long __force_order; static inline __attribute__((always_inline)) unsigned long native_read_cr0(void) { unsigned long val; asm volatile("mov %%cr0,%0\n\t" : "=r" (val), "=m" (__force_order)); return val; } static inline __attribute__((always_inline)) void native_write_cr0(unsigned long val) { asm volatile("mov %0,%%cr0": : "r" (val), "m" (__force_order)); } static inline __attribute__((always_inline)) unsigned long native_read_cr2(void) { unsigned long val; asm volatile("mov %%cr2,%0\n\t" : "=r" (val), "=m" (__force_order)); return val; } static inline __attribute__((always_inline)) void native_write_cr2(unsigned long val) { asm volatile("mov %0,%%cr2": : "r" (val), "m" (__force_order)); } static inline __attribute__((always_inline)) unsigned long native_read_cr3(void) { unsigned long val; asm volatile("mov %%cr3,%0\n\t" : "=r" (val), "=m" (__force_order)); return val; } static inline __attribute__((always_inline)) void native_write_cr3(unsigned long val) { asm volatile("mov %0,%%cr3": : "r" (val), "m" (__force_order)); } static inline __attribute__((always_inline)) unsigned long native_read_cr4(void) { unsigned long val; asm volatile("mov %%cr4,%0\n\t" : "=r" (val), "=m" (__force_order)); return val; } static inline __attribute__((always_inline)) unsigned long native_read_cr4_safe(void) { unsigned long val; asm volatile("1: mov %%cr4, %0\n" "2:\n" " .section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "1b" "," "2b" "\n" " .previous\n" : "=r" (val), "=m" (__force_order) : "0" (0)); return val; } static inline __attribute__((always_inline)) void native_write_cr4(unsigned long val) { asm volatile("mov %0,%%cr4": : "r" (val), "m" (__force_order)); } static inline __attribute__((always_inline)) void native_wbinvd(void) { asm volatile("wbinvd": : :"memory"); } static inline __attribute__((always_inline)) void clflush(volatile void *__p) { asm volatile("clflush %0" : "+m" (*(volatile char *)__p)); } void disable_hlt(void); void enable_hlt(void); void cpu_idle_wait(void); extern unsigned long arch_align_stack(unsigned long sp); extern void free_init_pages(char *what, unsigned long begin, unsigned long end); void default_idle(void); void stop_this_cpu(void *dummy); static inline __attribute__((always_inline)) __attribute__((always_inline)) void rdtsc_barrier(void) { asm volatile ("661:\n\t" ".byte " "0x8d,0x76,0x00" "\n" "\n662:\n" ".section .altinstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "661b\n" " " ".long" " " "663f\n" " .word " "(3*32+17)" "\n" " .byte 662b-661b\n" " .byte 664f-663f\n" ".previous\n" ".section .discard,\"aw\",@progbits\n" " .byte 0xff + (664f-663f) - (662b-661b)\n" ".previous\n" ".section .altinstr_replacement, \"ax\"\n" "663:\n\t" "mfence" "\n664:\n" ".previous" : : : "memory"); asm volatile ("661:\n\t" ".byte " "0x8d,0x76,0x00" "\n" "\n662:\n" ".section .altinstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "661b\n" " " ".long" " " "663f\n" " .word " "(3*32+18)" "\n" " .byte 662b-661b\n" " .byte 664f-663f\n" ".previous\n" ".section .discard,\"aw\",@progbits\n" " .byte 0xff + (664f-663f) - (662b-661b)\n" ".previous\n" ".section .altinstr_replacement, \"ax\"\n" "663:\n\t" "lfence" "\n664:\n" ".previous" : : : "memory"); } extern unsigned int __invalid_size_argument_for_IOC; extern cpumask_var_t cpu_callin_mask; extern cpumask_var_t cpu_callout_mask; extern cpumask_var_t cpu_initialized_mask; extern cpumask_var_t cpu_sibling_setup_mask; extern void setup_cpu_local_masks(void); struct msr { union { struct { u32 l; u32 h; }; u64 q; }; }; struct msr_info { u32 msr_no; struct msr reg; struct msr *msrs; int err; }; struct msr_regs_info { u32 *regs; int err; }; static inline __attribute__((always_inline)) unsigned long long native_read_tscp(unsigned int *aux) { unsigned long low, high; asm volatile(".byte 0x0f,0x01,0xf9" : "=a" (low), "=d" (high), "=c" (*aux)); return low | ((u64)high << 32); } static inline __attribute__((always_inline)) unsigned long long native_read_msr(unsigned int msr) { unsigned long long val; asm volatile("rdmsr" : "=A" (val) : "c" (msr)); return (val); } static inline __attribute__((always_inline)) unsigned long long native_read_msr_safe(unsigned int msr, int *err) { unsigned long long val; asm volatile("2: rdmsr ; xor %[err],%[err]\n" "1:\n\t" ".section .fixup,\"ax\"\n\t" "3: mov %[fault],%[err] ; jmp 1b\n\t" ".previous\n\t" " .section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "2b" "," "3b" "\n" " .previous\n" : [err] "=r" (*err), "=A" (val) : "c" (msr), [fault] "i" (-5)); return (val); } static inline __attribute__((always_inline)) void native_write_msr(unsigned int msr, unsigned low, unsigned high) { asm volatile("wrmsr" : : "c" (msr), "a"(low), "d" (high) : "memory"); } __attribute__((no_instrument_function)) static inline __attribute__((always_inline)) int native_write_msr_safe(unsigned int msr, unsigned low, unsigned high) { int err; asm volatile("2: wrmsr ; xor %[err],%[err]\n" "1:\n\t" ".section .fixup,\"ax\"\n\t" "3: mov %[fault],%[err] ; jmp 1b\n\t" ".previous\n\t" " .section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "2b" "," "3b" "\n" " .previous\n" : [err] "=a" (err) : "c" (msr), "0" (low), "d" (high), [fault] "i" (-5) : "memory"); return err; } extern unsigned long long native_read_tsc(void); extern int native_rdmsr_safe_regs(u32 regs[8]); extern int native_wrmsr_safe_regs(u32 regs[8]); static inline __attribute__((always_inline)) __attribute__((always_inline)) unsigned long long __native_read_tsc(void) { unsigned long long val; asm volatile("rdtsc" : "=A" (val)); return (val); } static inline __attribute__((always_inline)) unsigned long long native_read_pmc(int counter) { unsigned long long val; asm volatile("rdpmc" : "=A" (val) : "c" (counter)); return (val); } struct msr *msrs_alloc(void); void msrs_free(struct msr *msrs); int rdmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h); int wrmsr_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h); void rdmsr_on_cpus(const struct cpumask *mask, u32 msr_no, struct msr *msrs); void wrmsr_on_cpus(const struct cpumask *mask, u32 msr_no, struct msr *msrs); int rdmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 *l, u32 *h); int wrmsr_safe_on_cpu(unsigned int cpu, u32 msr_no, u32 l, u32 h); int rdmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]); int wrmsr_safe_regs_on_cpu(unsigned int cpu, u32 regs[8]); struct exec_domain; struct pt_regs; extern int register_exec_domain(struct exec_domain *); extern int unregister_exec_domain(struct exec_domain *); extern int __set_personality(unsigned int); enum { UNAME26 = 0x0020000, ADDR_NO_RANDOMIZE = 0x0040000, FDPIC_FUNCPTRS = 0x0080000, MMAP_PAGE_ZERO = 0x0100000, ADDR_COMPAT_LAYOUT = 0x0200000, READ_IMPLIES_EXEC = 0x0400000, ADDR_LIMIT_32BIT = 0x0800000, SHORT_INODE = 0x1000000, WHOLE_SECONDS = 0x2000000, STICKY_TIMEOUTS = 0x4000000, ADDR_LIMIT_3GB = 0x8000000, }; enum { PER_LINUX = 0x0000, PER_LINUX_32BIT = 0x0000 | ADDR_LIMIT_32BIT, PER_LINUX_FDPIC = 0x0000 | FDPIC_FUNCPTRS, PER_SVR4 = 0x0001 | STICKY_TIMEOUTS | MMAP_PAGE_ZERO, PER_SVR3 = 0x0002 | STICKY_TIMEOUTS | SHORT_INODE, PER_SCOSVR3 = 0x0003 | STICKY_TIMEOUTS | WHOLE_SECONDS | SHORT_INODE, PER_OSR5 = 0x0003 | STICKY_TIMEOUTS | WHOLE_SECONDS, PER_WYSEV386 = 0x0004 | STICKY_TIMEOUTS | SHORT_INODE, PER_ISCR4 = 0x0005 | STICKY_TIMEOUTS, PER_BSD = 0x0006, PER_SUNOS = 0x0006 | STICKY_TIMEOUTS, PER_XENIX = 0x0007 | STICKY_TIMEOUTS | SHORT_INODE, PER_LINUX32 = 0x0008, PER_LINUX32_3GB = 0x0008 | ADDR_LIMIT_3GB, PER_IRIX32 = 0x0009 | STICKY_TIMEOUTS, PER_IRIXN32 = 0x000a | STICKY_TIMEOUTS, PER_IRIX64 = 0x000b | STICKY_TIMEOUTS, PER_RISCOS = 0x000c, PER_SOLARIS = 0x000d | STICKY_TIMEOUTS, PER_UW7 = 0x000e | STICKY_TIMEOUTS | MMAP_PAGE_ZERO, PER_OSF4 = 0x000f, PER_HPUX = 0x0010, PER_MASK = 0x00ff, }; typedef void (*handler_t)(int, struct pt_regs *); struct exec_domain { const char *name; handler_t handler; unsigned char pers_low; unsigned char pers_high; unsigned long *signal_map; unsigned long *signal_invmap; struct map_segment *err_map; struct map_segment *socktype_map; struct map_segment *sockopt_map; struct map_segment *af_map; struct module *module; struct exec_domain *next; }; extern s64 div_s64_rem(s64 dividend, s32 divisor, s32 *remainder); extern u64 div64_u64(u64 dividend, u64 divisor); extern s64 div64_s64(s64 dividend, s64 divisor); static inline __attribute__((always_inline)) u64 div_u64(u64 dividend, u32 divisor) { u32 remainder; return div_u64_rem(dividend, divisor, &remainder); } static inline __attribute__((always_inline)) s64 div_s64(s64 dividend, s32 divisor) { s32 remainder; return div_s64_rem(dividend, divisor, &remainder); } u32 iter_div_u64_rem(u64 dividend, u32 divisor, u64 *remainder); static inline __attribute__((always_inline)) __attribute__((always_inline)) u32 __iter_div_u64_rem(u64 dividend, u32 divisor, u64 *remainder) { u32 ret = 0; while (dividend >= divisor) { asm("" : "+rm"(dividend)); dividend -= divisor; ret++; } *remainder = dividend; return ret; } static inline __attribute__((always_inline)) void * __attribute__((warn_unused_result)) ERR_PTR(long error) { return (void *) error; } static inline __attribute__((always_inline)) long __attribute__((warn_unused_result)) PTR_ERR(const void *ptr) { return (long) ptr; } static inline __attribute__((always_inline)) long __attribute__((warn_unused_result)) IS_ERR(const void *ptr) { return (__builtin_constant_p(((unsigned long)ptr) >= (unsigned long)-4095) ? !!(((unsigned long)ptr) >= (unsigned long)-4095) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/err.h", .line = 34, }; ______r = __builtin_expect(!!(((unsigned long)ptr) >= (unsigned long)-4095), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })); } static inline __attribute__((always_inline)) long __attribute__((warn_unused_result)) IS_ERR_OR_NULL(const void *ptr) { return !ptr || (__builtin_constant_p(((unsigned long)ptr) >= (unsigned long)-4095) ? !!(((unsigned long)ptr) >= (unsigned long)-4095) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/err.h", .line = 39, }; ______r = __builtin_expect(!!(((unsigned long)ptr) >= (unsigned long)-4095), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })); } static inline __attribute__((always_inline)) void * __attribute__((warn_unused_result)) ERR_CAST(const void *ptr) { return (void *) ptr; } static inline __attribute__((always_inline)) int __attribute__((warn_unused_result)) PTR_RET(const void *ptr) { if (__builtin_constant_p(((IS_ERR(ptr)))) ? !!((IS_ERR(ptr))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/err.h", .line = 57, }; ______r = !!((IS_ERR(ptr))); ______f.miss_hit[______r]++; ______r; })) return PTR_ERR(ptr); else return 0; } static inline __attribute__((always_inline)) void *current_text_addr(void) { void *pc; asm volatile("mov $1f, %0; 1:":"=r" (pc)); return pc; } struct cpuinfo_x86 { __u8 x86; __u8 x86_vendor; __u8 x86_model; __u8 x86_mask; char wp_works_ok; char hlt_works_ok; char hard_math; char rfu; char fdiv_bug; char f00f_bug; char coma_bug; char pad0; __u8 x86_virt_bits; __u8 x86_phys_bits; __u8 x86_coreid_bits; __u32 extended_cpuid_level; int cpuid_level; __u32 x86_capability[10]; char x86_vendor_id[16]; char x86_model_id[64]; int x86_cache_size; int x86_cache_alignment; int x86_power; unsigned long loops_per_jiffy; u16 x86_max_cores; u16 apicid; u16 initial_apicid; u16 x86_clflush_size; u16 booted_cores; u16 phys_proc_id; u16 cpu_core_id; u8 compute_unit_id; u16 cpu_index; } __attribute__((__aligned__((1 << (6))))); extern struct cpuinfo_x86 boot_cpu_data; extern struct cpuinfo_x86 new_cpu_data; extern struct tss_struct doublefault_tss; extern __u32 cpu_caps_cleared[10]; extern __u32 cpu_caps_set[10]; extern __attribute__((section(".data..percpu" ""))) __typeof__(struct cpuinfo_x86) cpu_info __attribute__((__aligned__((1 << (6))))); extern const struct seq_operations cpuinfo_op; static inline __attribute__((always_inline)) int hlt_works(int cpu) { return (*({ do { const void *__vpp_verify = (typeof((&(cpu_info))))((void *)0); (void)__vpp_verify; } while (0); ({ unsigned long __ptr; __asm__ ("" : "=r"(__ptr) : "0"((typeof(*(&(cpu_info))) *)(&(cpu_info)))); (typeof((typeof(*(&(cpu_info))) *)(&(cpu_info)))) (__ptr + (((__per_cpu_offset[cpu])))); }); })).hlt_works_ok; } extern void cpu_detect(struct cpuinfo_x86 *c); extern struct pt_regs *idle_regs(struct pt_regs *); extern void early_cpu_init(void); extern void identify_boot_cpu(void); extern void identify_secondary_cpu(struct cpuinfo_x86 *); extern void print_cpu_info(struct cpuinfo_x86 *); extern void init_scattered_cpuid_features(struct cpuinfo_x86 *c); extern unsigned int init_intel_cacheinfo(struct cpuinfo_x86 *c); extern unsigned short num_cache_leaves; extern void detect_extended_topology(struct cpuinfo_x86 *c); extern void detect_ht(struct cpuinfo_x86 *c); static inline __attribute__((always_inline)) void native_cpuid(unsigned int *eax, unsigned int *ebx, unsigned int *ecx, unsigned int *edx) { asm volatile("cpuid" : "=a" (*eax), "=b" (*ebx), "=c" (*ecx), "=d" (*edx) : "0" (*eax), "2" (*ecx)); } static inline __attribute__((always_inline)) void load_cr3(pgd_t *pgdir) { write_cr3((((unsigned long)(pgdir)) - ((unsigned long)(0xC0000000UL)))); } struct x86_hw_tss { unsigned short back_link, __blh; unsigned long sp0; unsigned short ss0, __ss0h; unsigned long sp1; unsigned short ss1, __ss1h; unsigned long sp2; unsigned short ss2, __ss2h; unsigned long __cr3; unsigned long ip; unsigned long flags; unsigned long ax; unsigned long cx; unsigned long dx; unsigned long bx; unsigned long sp; unsigned long bp; unsigned long si; unsigned long di; unsigned short es, __esh; unsigned short cs, __csh; unsigned short ss, __ssh; unsigned short ds, __dsh; unsigned short fs, __fsh; unsigned short gs, __gsh; unsigned short ldt, __ldth; unsigned short trace; unsigned short io_bitmap_base; } __attribute__((packed)); struct tss_struct { struct x86_hw_tss x86_tss; unsigned long io_bitmap[((65536/8)/sizeof(long)) + 1]; unsigned long stack[64]; } __attribute__((__aligned__((1 << (6))))); extern __attribute__((section(".data..percpu" ""))) __typeof__(struct tss_struct) init_tss __attribute__((__aligned__((1 << (6))))); struct orig_ist { unsigned long ist[7]; }; struct i387_fsave_struct { u32 cwd; u32 swd; u32 twd; u32 fip; u32 fcs; u32 foo; u32 fos; u32 st_space[20]; u32 status; }; struct i387_fxsave_struct { u16 cwd; u16 swd; u16 twd; u16 fop; union { struct { u64 rip; u64 rdp; }; struct { u32 fip; u32 fcs; u32 foo; u32 fos; }; }; u32 mxcsr; u32 mxcsr_mask; u32 st_space[32]; u32 xmm_space[64]; u32 padding[12]; union { u32 padding1[12]; u32 sw_reserved[12]; }; } __attribute__((aligned(16))); struct i387_soft_struct { u32 cwd; u32 swd; u32 twd; u32 fip; u32 fcs; u32 foo; u32 fos; u32 st_space[20]; u8 ftop; u8 changed; u8 lookahead; u8 no_update; u8 rm; u8 alimit; struct math_emu_info *info; u32 entry_eip; }; struct ymmh_struct { u32 ymmh_space[64]; }; struct xsave_hdr_struct { u64 xstate_bv; u64 reserved1[2]; u64 reserved2[5]; } __attribute__((packed)); struct xsave_struct { struct i387_fxsave_struct i387; struct xsave_hdr_struct xsave_hdr; struct ymmh_struct ymmh; } __attribute__ ((packed, aligned (64))); union thread_xstate { struct i387_fsave_struct fsave; struct i387_fxsave_struct fxsave; struct i387_soft_struct soft; struct xsave_struct xsave; }; struct fpu { union thread_xstate *state; }; struct stack_canary { char __pad[20]; unsigned long canary; }; extern __attribute__((section(".data..percpu" ""))) __typeof__(struct stack_canary) stack_canary __attribute__((__aligned__((1 << (6))))); extern unsigned int xstate_size; extern void free_thread_xstate(struct task_struct *); extern struct kmem_cache *task_xstate_cachep; struct perf_event; struct thread_struct { struct desc_struct tls_array[3]; unsigned long sp0; unsigned long sp; unsigned long sysenter_cs; unsigned long ip; unsigned long gs; struct perf_event *ptrace_bps[4]; unsigned long debugreg6; unsigned long ptrace_dr7; unsigned long cr2; unsigned long trap_no; unsigned long error_code; struct fpu fpu; struct vm86_struct *vm86_info; unsigned long screen_bitmap; unsigned long v86flags; unsigned long v86mask; unsigned long saved_sp0; unsigned int saved_fs; unsigned int saved_gs; unsigned long *io_bitmap_ptr; unsigned long iopl; unsigned io_bitmap_max; }; static inline __attribute__((always_inline)) unsigned long native_get_debugreg(int regno) { unsigned long val = 0; switch (regno) { case 0: asm("mov %%db0, %0" :"=r" (val)); break; case 1: asm("mov %%db1, %0" :"=r" (val)); break; case 2: asm("mov %%db2, %0" :"=r" (val)); break; case 3: asm("mov %%db3, %0" :"=r" (val)); break; case 6: asm("mov %%db6, %0" :"=r" (val)); break; case 7: asm("mov %%db7, %0" :"=r" (val)); break; default: do { asm volatile("1:\tud2\n" ".pushsection __bug_table,\"a\"\n" "2:\t.long 1b, %c0\n" "\t.word %c1, 0\n" "\t.org 2b+%c2\n" ".popsection" : : "i" ("/data/exp/linux-3.0.4/arch/x86/include/asm/processor.h"), "i" (499), "i" (sizeof(struct bug_entry))); __builtin_unreachable(); } while (0); } return val; } static inline __attribute__((always_inline)) void native_set_debugreg(int regno, unsigned long value) { switch (regno) { case 0: asm("mov %0, %%db0" ::"r" (value)); break; case 1: asm("mov %0, %%db1" ::"r" (value)); break; case 2: asm("mov %0, %%db2" ::"r" (value)); break; case 3: asm("mov %0, %%db3" ::"r" (value)); break; case 6: asm("mov %0, %%db6" ::"r" (value)); break; case 7: asm("mov %0, %%db7" ::"r" (value)); break; default: do { asm volatile("1:\tud2\n" ".pushsection __bug_table,\"a\"\n" "2:\t.long 1b, %c0\n" "\t.word %c1, 0\n" "\t.org 2b+%c2\n" ".popsection" : : "i" ("/data/exp/linux-3.0.4/arch/x86/include/asm/processor.h"), "i" (526), "i" (sizeof(struct bug_entry))); __builtin_unreachable(); } while (0); } } static inline __attribute__((always_inline)) void native_set_iopl_mask(unsigned mask) { unsigned int reg; asm volatile ("pushfl;" "popl %0;" "andl %1, %0;" "orl %2, %0;" "pushl %0;" "popfl" : "=&r" (reg) : "i" (~0x00003000), "r" (mask)); } static inline __attribute__((always_inline)) void native_load_sp0(struct tss_struct *tss, struct thread_struct *thread) { tss->x86_tss.sp0 = thread->sp0; if (__builtin_constant_p((((__builtin_constant_p(tss->x86_tss.ss1 != thread->sysenter_cs) ? !!(tss->x86_tss.ss1 != thread->sysenter_cs) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/processor.h", .line = 555, }; ______r = __builtin_expect(!!(tss->x86_tss.ss1 != thread->sysenter_cs), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(tss->x86_tss.ss1 != thread->sysenter_cs) ? !!(tss->x86_tss.ss1 != thread->sysenter_cs) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/processor.h", .line = 555, }; ______r = __builtin_expect(!!(tss->x86_tss.ss1 != thread->sysenter_cs), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/processor.h", .line = 555, }; ______r = !!(((__builtin_constant_p(tss->x86_tss.ss1 != thread->sysenter_cs) ? !!(tss->x86_tss.ss1 != thread->sysenter_cs) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/processor.h", .line = 555, }; ______r = __builtin_expect(!!(tss->x86_tss.ss1 != thread->sysenter_cs), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) { tss->x86_tss.ss1 = thread->sysenter_cs; do { paravirt_write_msr(0x00000174, thread->sysenter_cs, 0); } while (0); } } static inline __attribute__((always_inline)) void native_swapgs(void) { } extern unsigned long mmu_cr4_features; static inline __attribute__((always_inline)) void set_in_cr4(unsigned long mask) { unsigned long cr4; mmu_cr4_features |= mask; cr4 = read_cr4(); cr4 |= mask; write_cr4(cr4); } static inline __attribute__((always_inline)) void clear_in_cr4(unsigned long mask) { unsigned long cr4; mmu_cr4_features &= ~mask; cr4 = read_cr4(); cr4 &= ~mask; write_cr4(cr4); } typedef struct { unsigned long seg; } mm_segment_t; extern int kernel_thread(int (*fn)(void *), void *arg, unsigned long flags); extern void release_thread(struct task_struct *); extern void prepare_to_copy(struct task_struct *tsk); unsigned long get_wchan(struct task_struct *p); static inline __attribute__((always_inline)) void cpuid(unsigned int op, unsigned int *eax, unsigned int *ebx, unsigned int *ecx, unsigned int *edx) { *eax = op; *ecx = 0; __cpuid(eax, ebx, ecx, edx); } static inline __attribute__((always_inline)) void cpuid_count(unsigned int op, int count, unsigned int *eax, unsigned int *ebx, unsigned int *ecx, unsigned int *edx) { *eax = op; *ecx = count; __cpuid(eax, ebx, ecx, edx); } static inline __attribute__((always_inline)) unsigned int cpuid_eax(unsigned int op) { unsigned int eax, ebx, ecx, edx; cpuid(op, &eax, &ebx, &ecx, &edx); return eax; } static inline __attribute__((always_inline)) unsigned int cpuid_ebx(unsigned int op) { unsigned int eax, ebx, ecx, edx; cpuid(op, &eax, &ebx, &ecx, &edx); return ebx; } static inline __attribute__((always_inline)) unsigned int cpuid_ecx(unsigned int op) { unsigned int eax, ebx, ecx, edx; cpuid(op, &eax, &ebx, &ecx, &edx); return ecx; } static inline __attribute__((always_inline)) unsigned int cpuid_edx(unsigned int op) { unsigned int eax, ebx, ecx, edx; cpuid(op, &eax, &ebx, &ecx, &edx); return edx; } static inline __attribute__((always_inline)) void rep_nop(void) { asm volatile("rep; nop" ::: "memory"); } static inline __attribute__((always_inline)) void cpu_relax(void) { rep_nop(); } static inline __attribute__((always_inline)) void sync_core(void) { int tmp; asm volatile("cpuid" : "=a" (tmp) : "0" (1) : "ebx", "ecx", "edx", "memory"); } static inline __attribute__((always_inline)) void __monitor(const void *eax, unsigned long ecx, unsigned long edx) { asm volatile(".byte 0x0f, 0x01, 0xc8;" :: "a" (eax), "c" (ecx), "d"(edx)); } static inline __attribute__((always_inline)) void __mwait(unsigned long eax, unsigned long ecx) { asm volatile(".byte 0x0f, 0x01, 0xc9;" :: "a" (eax), "c" (ecx)); } static inline __attribute__((always_inline)) void __sti_mwait(unsigned long eax, unsigned long ecx) { trace_hardirqs_on(); asm volatile("sti; .byte 0x0f, 0x01, 0xc9;" :: "a" (eax), "c" (ecx)); } extern void mwait_idle_with_hints(unsigned long eax, unsigned long ecx); extern void select_idle_routine(const struct cpuinfo_x86 *c); extern void init_amd_e400_c1e_mask(void); extern unsigned long boot_option_idle_override; extern bool amd_e400_c1e_detected; enum idle_boot_override {IDLE_NO_OVERRIDE=0, IDLE_HALT, IDLE_NOMWAIT, IDLE_POLL, IDLE_FORCE_MWAIT}; extern void enable_sep_cpu(void); extern int sysenter_setup(void); extern void early_trap_init(void); extern struct desc_ptr early_gdt_descr; extern void cpu_set_gdt(int); extern void switch_to_new_gdt(int); extern void load_percpu_segment(int); extern void cpu_init(void); static inline __attribute__((always_inline)) unsigned long get_debugctlmsr(void) { unsigned long debugctlmsr = 0; do { int _err; debugctlmsr = paravirt_read_msr(0x000001d9, &_err); } while (0); return debugctlmsr; } static inline __attribute__((always_inline)) void update_debugctlmsr(unsigned long debugctlmsr) { do { paravirt_write_msr(0x000001d9, (u32)((u64)(debugctlmsr)), ((u64)(debugctlmsr))>>32); } while (0); } extern unsigned int machine_id; extern unsigned int machine_submodel_id; extern unsigned int BIOS_revision; extern int bootloader_type; extern int bootloader_version; extern char ignore_fpu_irq; static inline __attribute__((always_inline)) void prefetch(const void *x) { asm volatile ("661:\n\t" ".byte " "0x8d,0x74,0x26,0x00" "\n" "\n662:\n" ".section .altinstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "661b\n" " " ".long" " " "663f\n" " .word " "(0*32+25)" "\n" " .byte 662b-661b\n" " .byte 664f-663f\n" ".previous\n" ".section .discard,\"aw\",@progbits\n" " .byte 0xff + (664f-663f) - (662b-661b)\n" ".previous\n" ".section .altinstr_replacement, \"ax\"\n" "663:\n\t" "prefetchnta (%1)" "\n664:\n" ".previous" : : "i" (0), "r" (x)) ; } static inline __attribute__((always_inline)) void prefetchw(const void *x) { asm volatile ("661:\n\t" ".byte " "0x8d,0x74,0x26,0x00" "\n" "\n662:\n" ".section .altinstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "661b\n" " " ".long" " " "663f\n" " .word " "(1*32+31)" "\n" " .byte 662b-661b\n" " .byte 664f-663f\n" ".previous\n" ".section .discard,\"aw\",@progbits\n" " .byte 0xff + (664f-663f) - (662b-661b)\n" ".previous\n" ".section .altinstr_replacement, \"ax\"\n" "663:\n\t" "prefetchw (%1)" "\n664:\n" ".previous" : : "i" (0), "r" (x)) ; } static inline __attribute__((always_inline)) void spin_lock_prefetch(const void *x) { prefetchw(x); } extern unsigned long thread_saved_pc(struct task_struct *tsk); extern void start_thread(struct pt_regs *regs, unsigned long new_ip, unsigned long new_sp); extern int get_tsc_mode(unsigned long adr); extern int set_tsc_mode(unsigned int val); extern int amd_get_nb_id(int cpu); struct aperfmperf { u64 aperf, mperf; }; static inline __attribute__((always_inline)) void get_aperfmperf(struct aperfmperf *am) { ({ static bool __warned; int __ret_warn_once = !!(!(__builtin_constant_p((3*32+28)) && ( ((((3*32+28))>>5)==0 && (1UL<<(((3*32+28))&31) & ((1<<((0*32+ 0) & 31))|0|0|(1<<((0*32+ 6) & 31))| (1<<((0*32+ 8) & 31))|0|0|(1<<((0*32+15) & 31))| 0|0))) || ((((3*32+28))>>5)==1 && (1UL<<(((3*32+28))&31) & (0|0))) || ((((3*32+28))>>5)==2 && (1UL<<(((3*32+28))&31) & 0)) || ((((3*32+28))>>5)==3 && (1UL<<(((3*32+28))&31) & (0))) || ((((3*32+28))>>5)==4 && (1UL<<(((3*32+28))&31) & 0)) || ((((3*32+28))>>5)==5 && (1UL<<(((3*32+28))&31) & 0)) || ((((3*32+28))>>5)==6 && (1UL<<(((3*32+28))&31) & 0)) || ((((3*32+28))>>5)==7 && (1UL<<(((3*32+28))&31) & 0)) || ((((3*32+28))>>5)==8 && (1UL<<(((3*32+28))&31) & 0)) || ((((3*32+28))>>5)==9 && (1UL<<(((3*32+28))&31) & 0)) ) ? 1 : (__builtin_constant_p(((3*32+28))) ? constant_test_bit(((3*32+28)), ((unsigned long *)((&boot_cpu_data)->x86_capability))) : variable_test_bit(((3*32+28)), ((unsigned long *)((&boot_cpu_data)->x86_capability)))))); if (__builtin_constant_p((((__builtin_constant_p(__ret_warn_once) ? !!(__ret_warn_once) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/processor.h", .line = 981, }; ______r = __builtin_expect(!!(__ret_warn_once), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(__ret_warn_once) ? !!(__ret_warn_once) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/processor.h", .line = 981, }; ______r = __builtin_expect(!!(__ret_warn_once), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/processor.h", .line = 981, }; ______r = !!(((__builtin_constant_p(__ret_warn_once) ? !!(__ret_warn_once) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/processor.h", .line = 981, }; ______r = __builtin_expect(!!(__ret_warn_once), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) if (__builtin_constant_p(((({ int __ret_warn_on = !!(!__warned); if (__builtin_constant_p((((__builtin_constant_p(__ret_warn_on) ? !!(__ret_warn_on) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/processor.h", .line = 981, }; ______r = __builtin_expect(!!(__ret_warn_on), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(__ret_warn_on) ? !!(__ret_warn_on) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/processor.h", .line = 981, }; ______r = __builtin_expect(!!(__ret_warn_on), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/processor.h", .line = 981, }; ______r = !!(((__builtin_constant_p(__ret_warn_on) ? !!(__ret_warn_on) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/processor.h", .line = 981, }; ______r = __builtin_expect(!!(__ret_warn_on), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) warn_slowpath_null("/data/exp/linux-3.0.4/arch/x86/include/asm/processor.h", 981); (__builtin_constant_p(__ret_warn_on) ? !!(__ret_warn_on) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/processor.h", .line = 981, }; ______r = __builtin_expect(!!(__ret_warn_on), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })); })))) ? !!((({ int __ret_warn_on = !!(!__warned); if (__builtin_constant_p((((__builtin_constant_p(__ret_warn_on) ? !!(__ret_warn_on) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/processor.h", .line = 981, }; ______r = __builtin_expect(!!(__ret_warn_on), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(__ret_warn_on) ? !!(__ret_warn_on) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/processor.h", .line = 981, }; ______r = __builtin_expect(!!(__ret_warn_on), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/processor.h", .line = 981, }; ______r = !!(((__builtin_constant_p(__ret_warn_on) ? !!(__ret_warn_on) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/processor.h", .line = 981, }; ______r = __builtin_expect(!!(__ret_warn_on), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) warn_slowpath_null("/data/exp/linux-3.0.4/arch/x86/include/asm/processor.h", 981); (__builtin_constant_p(__ret_warn_on) ? !!(__ret_warn_on) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/processor.h", .line = 981, }; ______r = __builtin_expect(!!(__ret_warn_on), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })); }))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/processor.h", .line = 981, }; ______r = !!((({ int __ret_warn_on = !!(!__warned); if (__builtin_constant_p((((__builtin_constant_p(__ret_warn_on) ? !!(__ret_warn_on) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/processor.h", .line = 981, }; ______r = __builtin_expect(!!(__ret_warn_on), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(__ret_warn_on) ? !!(__ret_warn_on) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/processor.h", .line = 981, }; ______r = __builtin_expect(!!(__ret_warn_on), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/processor.h", .line = 981, }; ______r = !!(((__builtin_constant_p(__ret_warn_on) ? !!(__ret_warn_on) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/processor.h", .line = 981, }; ______r = __builtin_expect(!!(__ret_warn_on), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) warn_slowpath_null("/data/exp/linux-3.0.4/arch/x86/include/asm/processor.h", 981); (__builtin_constant_p(__ret_warn_on) ? !!(__ret_warn_on) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/processor.h", .line = 981, }; ______r = __builtin_expect(!!(__ret_warn_on), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })); }))); ______f.miss_hit[______r]++; ______r; })) __warned = true; (__builtin_constant_p(__ret_warn_once) ? !!(__ret_warn_once) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/processor.h", .line = 981, }; ______r = __builtin_expect(!!(__ret_warn_once), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })); }); do { int _err; am->aperf = paravirt_read_msr(0x000000e8, &_err); } while (0); do { int _err; am->mperf = paravirt_read_msr(0x000000e7, &_err); } while (0); } static inline __attribute__((always_inline)) unsigned long calc_aperfmperf_ratio(struct aperfmperf *old, struct aperfmperf *new) { u64 aperf = new->aperf - old->aperf; u64 mperf = new->mperf - old->mperf; unsigned long ratio = aperf; mperf >>= 10; if (__builtin_constant_p(((mperf))) ? !!((mperf)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/processor.h", .line = 998, }; ______r = !!((mperf)); ______f.miss_hit[______r]++; ______r; })) ratio = div64_u64(aperf, mperf); return ratio; } extern const int amd_erratum_383[]; extern const int amd_erratum_400[]; extern bool cpu_has_amd_erratum(const int *); extern void mcount(void); static inline __attribute__((always_inline)) unsigned long ftrace_call_adjust(unsigned long addr) { return addr; } struct dyn_arch_ftrace { }; static inline __attribute__((always_inline)) int atomic_read(const atomic_t *v) { return (*(volatile int *)&(v)->counter); } static inline __attribute__((always_inline)) void atomic_set(atomic_t *v, int i) { v->counter = i; } static inline __attribute__((always_inline)) void atomic_add(int i, atomic_t *v) { asm volatile(".section .smp_locks,\"a\"\n" ".balign 4\n" ".long 671f - .\n" ".previous\n" "671:" "\n\tlock; " "addl %1,%0" : "+m" (v->counter) : "ir" (i)); } static inline __attribute__((always_inline)) void atomic_sub(int i, atomic_t *v) { asm volatile(".section .smp_locks,\"a\"\n" ".balign 4\n" ".long 671f - .\n" ".previous\n" "671:" "\n\tlock; " "subl %1,%0" : "+m" (v->counter) : "ir" (i)); } static inline __attribute__((always_inline)) int atomic_sub_and_test(int i, atomic_t *v) { unsigned char c; asm volatile(".section .smp_locks,\"a\"\n" ".balign 4\n" ".long 671f - .\n" ".previous\n" "671:" "\n\tlock; " "subl %2,%0; sete %1" : "+m" (v->counter), "=qm" (c) : "ir" (i) : "memory"); return c; } static inline __attribute__((always_inline)) void atomic_inc(atomic_t *v) { asm volatile(".section .smp_locks,\"a\"\n" ".balign 4\n" ".long 671f - .\n" ".previous\n" "671:" "\n\tlock; " "incl %0" : "+m" (v->counter)); } static inline __attribute__((always_inline)) void atomic_dec(atomic_t *v) { asm volatile(".section .smp_locks,\"a\"\n" ".balign 4\n" ".long 671f - .\n" ".previous\n" "671:" "\n\tlock; " "decl %0" : "+m" (v->counter)); } static inline __attribute__((always_inline)) int atomic_dec_and_test(atomic_t *v) { unsigned char c; asm volatile(".section .smp_locks,\"a\"\n" ".balign 4\n" ".long 671f - .\n" ".previous\n" "671:" "\n\tlock; " "decl %0; sete %1" : "+m" (v->counter), "=qm" (c) : : "memory"); return c != 0; } static inline __attribute__((always_inline)) int atomic_inc_and_test(atomic_t *v) { unsigned char c; asm volatile(".section .smp_locks,\"a\"\n" ".balign 4\n" ".long 671f - .\n" ".previous\n" "671:" "\n\tlock; " "incl %0; sete %1" : "+m" (v->counter), "=qm" (c) : : "memory"); return c != 0; } static inline __attribute__((always_inline)) int atomic_add_negative(int i, atomic_t *v) { unsigned char c; asm volatile(".section .smp_locks,\"a\"\n" ".balign 4\n" ".long 671f - .\n" ".previous\n" "671:" "\n\tlock; " "addl %2,%0; sets %1" : "+m" (v->counter), "=qm" (c) : "ir" (i) : "memory"); return c; } static inline __attribute__((always_inline)) int atomic_add_return(int i, atomic_t *v) { int __i; __i = i; asm volatile(".section .smp_locks,\"a\"\n" ".balign 4\n" ".long 671f - .\n" ".previous\n" "671:" "\n\tlock; " "xaddl %0, %1" : "+r" (i), "+m" (v->counter) : : "memory"); return i + __i; } static inline __attribute__((always_inline)) int atomic_sub_return(int i, atomic_t *v) { return atomic_add_return(-i, v); } static inline __attribute__((always_inline)) int atomic_cmpxchg(atomic_t *v, int old, int new) { return ({ __typeof__(*(((&v->counter)))) __ret; __typeof__(*(((&v->counter)))) __old = (((old))); __typeof__(*(((&v->counter)))) __new = (((new))); switch ((sizeof(*&v->counter))) { case 1: { volatile u8 *__ptr = (volatile u8 *)(((&v->counter))); asm volatile(".section .smp_locks,\"a\"\n" ".balign 4\n" ".long 671f - .\n" ".previous\n" "671:" "\n\tlock; " "cmpxchgb %2,%1" : "=a" (__ret), "+m" (*__ptr) : "q" (__new), "0" (__old) : "memory"); break; } case 2: { volatile u16 *__ptr = (volatile u16 *)(((&v->counter))); asm volatile(".section .smp_locks,\"a\"\n" ".balign 4\n" ".long 671f - .\n" ".previous\n" "671:" "\n\tlock; " "cmpxchgw %2,%1" : "=a" (__ret), "+m" (*__ptr) : "r" (__new), "0" (__old) : "memory"); break; } case 4: { volatile u32 *__ptr = (volatile u32 *)(((&v->counter))); asm volatile(".section .smp_locks,\"a\"\n" ".balign 4\n" ".long 671f - .\n" ".previous\n" "671:" "\n\tlock; " "cmpxchgl %2,%1" : "=a" (__ret), "+m" (*__ptr) : "r" (__new), "0" (__old) : "memory"); break; } default: __cmpxchg_wrong_size(); } __ret; }); } static inline __attribute__((always_inline)) int atomic_xchg(atomic_t *v, int new) { return ({ __typeof(*((&v->counter))) __x = ((new)); switch (sizeof(*&v->counter)) { case 1: { volatile u8 *__ptr = (volatile u8 *)((&v->counter)); asm volatile("xchgb %0,%1" : "=q" (__x), "+m" (*__ptr) : "0" (__x) : "memory"); break; } case 2: { volatile u16 *__ptr = (volatile u16 *)((&v->counter)); asm volatile("xchgw %0,%1" : "=r" (__x), "+m" (*__ptr) : "0" (__x) : "memory"); break; } case 4: { volatile u32 *__ptr = (volatile u32 *)((&v->counter)); asm volatile("xchgl %0,%1" : "=r" (__x), "+m" (*__ptr) : "0" (__x) : "memory"); break; } default: __xchg_wrong_size(); } __x; }); } static inline __attribute__((always_inline)) int atomic_add_unless(atomic_t *v, int a, int u) { int c, old; c = atomic_read(v); for (;;) { if (__builtin_constant_p((((__builtin_constant_p(c == (u)) ? !!(c == (u)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/atomic.h", .line = 237, }; ______r = __builtin_expect(!!(c == (u)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(c == (u)) ? !!(c == (u)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/atomic.h", .line = 237, }; ______r = __builtin_expect(!!(c == (u)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/atomic.h", .line = 237, }; ______r = !!(((__builtin_constant_p(c == (u)) ? !!(c == (u)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/atomic.h", .line = 237, }; ______r = __builtin_expect(!!(c == (u)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) break; old = atomic_cmpxchg((v), c, c + (a)); if (__builtin_constant_p((((__builtin_constant_p(old == c) ? !!(old == c) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/atomic.h", .line = 240, }; ______r = __builtin_expect(!!(old == c), 1); ftrace_likely_update(&______f, ______r, 1); ______r; }))))) ? !!(((__builtin_constant_p(old == c) ? !!(old == c) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/atomic.h", .line = 240, }; ______r = __builtin_expect(!!(old == c), 1); ftrace_likely_update(&______f, ______r, 1); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/atomic.h", .line = 240, }; ______r = !!(((__builtin_constant_p(old == c) ? !!(old == c) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/atomic.h", .line = 240, }; ______r = __builtin_expect(!!(old == c), 1); ftrace_likely_update(&______f, ______r, 1); ______r; })))); ______f.miss_hit[______r]++; ______r; })) break; c = old; } return c != (u); } static inline __attribute__((always_inline)) int atomic_dec_if_positive(atomic_t *v) { int c, old, dec; c = atomic_read(v); for (;;) { dec = c - 1; if (__builtin_constant_p((((__builtin_constant_p(dec < 0) ? !!(dec < 0) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/atomic.h", .line = 262, }; ______r = __builtin_expect(!!(dec < 0), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(dec < 0) ? !!(dec < 0) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/atomic.h", .line = 262, }; ______r = __builtin_expect(!!(dec < 0), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/atomic.h", .line = 262, }; ______r = !!(((__builtin_constant_p(dec < 0) ? !!(dec < 0) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/atomic.h", .line = 262, }; ______r = __builtin_expect(!!(dec < 0), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) break; old = atomic_cmpxchg((v), c, dec); if (__builtin_constant_p((((__builtin_constant_p(old == c) ? !!(old == c) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/atomic.h", .line = 265, }; ______r = __builtin_expect(!!(old == c), 1); ftrace_likely_update(&______f, ______r, 1); ______r; }))))) ? !!(((__builtin_constant_p(old == c) ? !!(old == c) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/atomic.h", .line = 265, }; ______r = __builtin_expect(!!(old == c), 1); ftrace_likely_update(&______f, ______r, 1); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/atomic.h", .line = 265, }; ______r = !!(((__builtin_constant_p(old == c) ? !!(old == c) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/atomic.h", .line = 265, }; ______r = __builtin_expect(!!(old == c), 1); ftrace_likely_update(&______f, ______r, 1); ______r; })))); ______f.miss_hit[______r]++; ______r; })) break; c = old; } return dec; } static inline __attribute__((always_inline)) short int atomic_inc_short(short int *v) { asm(".section .smp_locks,\"a\"\n" ".balign 4\n" ".long 671f - .\n" ".previous\n" "671:" "\n\tlock; " "addw $1, %0" : "+m" (*v)); return *v; } typedef struct { u64 __attribute__((aligned(8))) counter; } atomic64_t; static inline __attribute__((always_inline)) long long atomic64_cmpxchg(atomic64_t *v, long long o, long long n) { return ((__typeof__(*(&v->counter)))__cmpxchg64((&v->counter), (unsigned long long)(o), (unsigned long long)(n))); } static inline __attribute__((always_inline)) long long atomic64_xchg(atomic64_t *v, long long n) { long long o; unsigned high = (unsigned)(n >> 32); unsigned low = (unsigned)n; asm volatile("call atomic64_" "xchg" "_cx8" : "=A" (o), "+b" (low), "+c" (high) : "S" (v) : "memory" ); return o; } static inline __attribute__((always_inline)) void atomic64_set(atomic64_t *v, long long i) { unsigned high = (unsigned)(i >> 32); unsigned low = (unsigned)i; asm volatile("call atomic64_" "set" "_cx8" : "+b" (low), "+c" (high) : "S" (v) : "eax", "edx", "memory" ); } static inline __attribute__((always_inline)) long long atomic64_read(atomic64_t *v) { long long r; asm volatile("call atomic64_" "read" "_cx8" : "=A" (r), "+c" (v) : : "memory" ); return r; } static inline __attribute__((always_inline)) long long atomic64_add_return(long long i, atomic64_t *v) { asm volatile("call atomic64_" "add_return" "_cx8" : "+A" (i), "+c" (v) : : "memory" ); return i; } static inline __attribute__((always_inline)) long long atomic64_sub_return(long long i, atomic64_t *v) { asm volatile("call atomic64_" "sub_return" "_cx8" : "+A" (i), "+c" (v) : : "memory" ); return i; } static inline __attribute__((always_inline)) long long atomic64_inc_return(atomic64_t *v) { long long a; asm volatile("call atomic64_" "inc_return" "_cx8" : "=A" (a) : "S" (v) : "memory", "ecx" ); return a; } static inline __attribute__((always_inline)) long long atomic64_dec_return(atomic64_t *v) { long long a; asm volatile("call atomic64_" "dec_return" "_cx8" : "=A" (a) : "S" (v) : "memory", "ecx" ); return a; } static inline __attribute__((always_inline)) long long atomic64_add(long long i, atomic64_t *v) { asm volatile("call atomic64_" "add_return" "_cx8" : "+A" (i), "+c" (v) : : "memory" ); return i; } static inline __attribute__((always_inline)) long long atomic64_sub(long long i, atomic64_t *v) { asm volatile("call atomic64_" "sub_return" "_cx8" : "+A" (i), "+c" (v) : : "memory" ); return i; } static inline __attribute__((always_inline)) int atomic64_sub_and_test(long long i, atomic64_t *v) { return atomic64_sub_return(i, v) == 0; } static inline __attribute__((always_inline)) void atomic64_inc(atomic64_t *v) { asm volatile("call atomic64_" "inc_return" "_cx8" : : "S" (v) : "memory", "eax", "ecx", "edx" ); } static inline __attribute__((always_inline)) void atomic64_dec(atomic64_t *v) { asm volatile("call atomic64_" "dec_return" "_cx8" : : "S" (v) : "memory", "eax", "ecx", "edx" ); } static inline __attribute__((always_inline)) int atomic64_dec_and_test(atomic64_t *v) { return atomic64_dec_return(v) == 0; } static inline __attribute__((always_inline)) int atomic64_inc_and_test(atomic64_t *v) { return atomic64_inc_return(v) == 0; } static inline __attribute__((always_inline)) int atomic64_add_negative(long long i, atomic64_t *v) { return atomic64_add_return(i, v) < 0; } static inline __attribute__((always_inline)) int atomic64_add_unless(atomic64_t *v, long long a, long long u) { unsigned low = (unsigned)u; unsigned high = (unsigned)(u >> 32); asm volatile("call atomic64_" "add_unless" "_cx8" "\n\t" : "+A" (a), "+c" (v), "+S" (low), "+D" (high) : : "memory"); return (int)a; } static inline __attribute__((always_inline)) int atomic64_inc_not_zero(atomic64_t *v) { int r; asm volatile("call atomic64_" "inc_not_zero" "_cx8" : "=a" (r) : "S" (v) : "ecx", "edx", "memory" ); return r; } static inline __attribute__((always_inline)) long long atomic64_dec_if_positive(atomic64_t *v) { long long r; asm volatile("call atomic64_" "dec_if_positive" "_cx8" : "=A" (r) : "S" (v) : "ecx", "memory" ); return r; } typedef atomic_t atomic_long_t; static inline __attribute__((always_inline)) long atomic_long_read(atomic_long_t *l) { atomic_t *v = (atomic_t *)l; return (long)atomic_read(v); } static inline __attribute__((always_inline)) void atomic_long_set(atomic_long_t *l, long i) { atomic_t *v = (atomic_t *)l; atomic_set(v, i); } static inline __attribute__((always_inline)) void atomic_long_inc(atomic_long_t *l) { atomic_t *v = (atomic_t *)l; atomic_inc(v); } static inline __attribute__((always_inline)) void atomic_long_dec(atomic_long_t *l) { atomic_t *v = (atomic_t *)l; atomic_dec(v); } static inline __attribute__((always_inline)) void atomic_long_add(long i, atomic_long_t *l) { atomic_t *v = (atomic_t *)l; atomic_add(i, v); } static inline __attribute__((always_inline)) void atomic_long_sub(long i, atomic_long_t *l) { atomic_t *v = (atomic_t *)l; atomic_sub(i, v); } static inline __attribute__((always_inline)) int atomic_long_sub_and_test(long i, atomic_long_t *l) { atomic_t *v = (atomic_t *)l; return atomic_sub_and_test(i, v); } static inline __attribute__((always_inline)) int atomic_long_dec_and_test(atomic_long_t *l) { atomic_t *v = (atomic_t *)l; return atomic_dec_and_test(v); } static inline __attribute__((always_inline)) int atomic_long_inc_and_test(atomic_long_t *l) { atomic_t *v = (atomic_t *)l; return atomic_inc_and_test(v); } static inline __attribute__((always_inline)) int atomic_long_add_negative(long i, atomic_long_t *l) { atomic_t *v = (atomic_t *)l; return atomic_add_negative(i, v); } static inline __attribute__((always_inline)) long atomic_long_add_return(long i, atomic_long_t *l) { atomic_t *v = (atomic_t *)l; return (long)atomic_add_return(i, v); } static inline __attribute__((always_inline)) long atomic_long_sub_return(long i, atomic_long_t *l) { atomic_t *v = (atomic_t *)l; return (long)atomic_sub_return(i, v); } static inline __attribute__((always_inline)) long atomic_long_inc_return(atomic_long_t *l) { atomic_t *v = (atomic_t *)l; return (long)(atomic_add_return(1, v)); } static inline __attribute__((always_inline)) long atomic_long_dec_return(atomic_long_t *l) { atomic_t *v = (atomic_t *)l; return (long)(atomic_sub_return(1, v)); } static inline __attribute__((always_inline)) long atomic_long_add_unless(atomic_long_t *l, long a, long u) { atomic_t *v = (atomic_t *)l; return (long)atomic_add_unless(v, a, u); } struct thread_info { struct task_struct *task; struct exec_domain *exec_domain; __u32 flags; __u32 status; __u32 cpu; int preempt_count; mm_segment_t addr_limit; struct restart_block restart_block; void *sysenter_return; unsigned long previous_esp; __u8 supervisor_stack[0]; int uaccess_err; }; register unsigned long current_stack_pointer asm("esp") __attribute__((__used__)); static inline __attribute__((always_inline)) struct thread_info *current_thread_info(void) { return (struct thread_info *) (current_stack_pointer & ~((((1UL) << 12) << 1) - 1)); } static inline __attribute__((always_inline)) void set_restore_sigmask(void) { struct thread_info *ti = current_thread_info(); ti->status |= 0x0008; set_bit(2, (unsigned long *)&ti->flags); } extern void arch_task_cache_init(void); extern void free_thread_info(struct thread_info *ti); extern int arch_dup_task_struct(struct task_struct *dst, struct task_struct *src); static inline __attribute__((always_inline)) void set_ti_thread_flag(struct thread_info *ti, int flag) { set_bit(flag, (unsigned long *)&ti->flags); } static inline __attribute__((always_inline)) void clear_ti_thread_flag(struct thread_info *ti, int flag) { clear_bit(flag, (unsigned long *)&ti->flags); } static inline __attribute__((always_inline)) int test_and_set_ti_thread_flag(struct thread_info *ti, int flag) { return test_and_set_bit(flag, (unsigned long *)&ti->flags); } static inline __attribute__((always_inline)) int test_and_clear_ti_thread_flag(struct thread_info *ti, int flag) { return test_and_clear_bit(flag, (unsigned long *)&ti->flags); } static inline __attribute__((always_inline)) int test_ti_thread_flag(struct thread_info *ti, int flag) { return (__builtin_constant_p((flag)) ? constant_test_bit((flag), ((unsigned long *)&ti->flags)) : variable_test_bit((flag), ((unsigned long *)&ti->flags))); } static inline __attribute__((always_inline)) void INIT_LIST_HEAD(struct list_head *list) { list->next = list; list->prev = list; } static inline __attribute__((always_inline)) void __list_add(struct list_head *new, struct list_head *prev, struct list_head *next) { next->prev = new; new->next = next; new->prev = prev; prev->next = new; } static inline __attribute__((always_inline)) void list_add(struct list_head *new, struct list_head *head) { __list_add(new, head, head->next); } static inline __attribute__((always_inline)) void list_add_tail(struct list_head *new, struct list_head *head) { __list_add(new, head->prev, head); } static inline __attribute__((always_inline)) void __list_del(struct list_head * prev, struct list_head * next) { next->prev = prev; prev->next = next; } static inline __attribute__((always_inline)) void __list_del_entry(struct list_head *entry) { __list_del(entry->prev, entry->next); } static inline __attribute__((always_inline)) void list_del(struct list_head *entry) { __list_del(entry->prev, entry->next); entry->next = ((void *) 0x00100100 + (0x0UL)); entry->prev = ((void *) 0x00200200 + (0x0UL)); } static inline __attribute__((always_inline)) void list_replace(struct list_head *old, struct list_head *new) { new->next = old->next; new->next->prev = new; new->prev = old->prev; new->prev->next = new; } static inline __attribute__((always_inline)) void list_replace_init(struct list_head *old, struct list_head *new) { list_replace(old, new); INIT_LIST_HEAD(old); } static inline __attribute__((always_inline)) void list_del_init(struct list_head *entry) { __list_del_entry(entry); INIT_LIST_HEAD(entry); } static inline __attribute__((always_inline)) void list_move(struct list_head *list, struct list_head *head) { __list_del_entry(list); list_add(list, head); } static inline __attribute__((always_inline)) void list_move_tail(struct list_head *list, struct list_head *head) { __list_del_entry(list); list_add_tail(list, head); } static inline __attribute__((always_inline)) int list_is_last(const struct list_head *list, const struct list_head *head) { return list->next == head; } static inline __attribute__((always_inline)) int list_empty(const struct list_head *head) { return head->next == head; } static inline __attribute__((always_inline)) int list_empty_careful(const struct list_head *head) { struct list_head *next = head->next; return (next == head) && (next == head->prev); } static inline __attribute__((always_inline)) void list_rotate_left(struct list_head *head) { struct list_head *first; if (__builtin_constant_p(((!list_empty(head)))) ? !!((!list_empty(head))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/list.h", .line = 218, }; ______r = !!((!list_empty(head))); ______f.miss_hit[______r]++; ______r; })) { first = head->next; list_move_tail(first, head); } } static inline __attribute__((always_inline)) int list_is_singular(const struct list_head *head) { return !list_empty(head) && (head->next == head->prev); } static inline __attribute__((always_inline)) void __list_cut_position(struct list_head *list, struct list_head *head, struct list_head *entry) { struct list_head *new_first = entry->next; list->next = head->next; list->next->prev = list; list->prev = entry; entry->next = list; head->next = new_first; new_first->prev = head; } static inline __attribute__((always_inline)) void list_cut_position(struct list_head *list, struct list_head *head, struct list_head *entry) { if (__builtin_constant_p(((list_empty(head)))) ? !!((list_empty(head))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/list.h", .line = 262, }; ______r = !!((list_empty(head))); ______f.miss_hit[______r]++; ______r; })) return; if (__builtin_constant_p(((list_is_singular(head) && (head->next != entry && head != entry)))) ? !!((list_is_singular(head) && (head->next != entry && head != entry))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/list.h" , .line = 265 , }; ______r = !!((list_is_singular(head) && (head->next != entry && head != entry))); ______f.miss_hit[______r]++; ______r; })) return; if (__builtin_constant_p(((entry == head))) ? !!((entry == head)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/list.h", .line = 267, }; ______r = !!((entry == head)); ______f.miss_hit[______r]++; ______r; })) INIT_LIST_HEAD(list); else __list_cut_position(list, head, entry); } static inline __attribute__((always_inline)) void __list_splice(const struct list_head *list, struct list_head *prev, struct list_head *next) { struct list_head *first = list->next; struct list_head *last = list->prev; first->prev = prev; prev->next = first; last->next = next; next->prev = last; } static inline __attribute__((always_inline)) void list_splice(const struct list_head *list, struct list_head *head) { if (__builtin_constant_p(((!list_empty(list)))) ? !!((!list_empty(list))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/list.h", .line = 295, }; ______r = !!((!list_empty(list))); ______f.miss_hit[______r]++; ______r; })) __list_splice(list, head, head->next); } static inline __attribute__((always_inline)) void list_splice_tail(struct list_head *list, struct list_head *head) { if (__builtin_constant_p(((!list_empty(list)))) ? !!((!list_empty(list))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/list.h", .line = 307, }; ______r = !!((!list_empty(list))); ______f.miss_hit[______r]++; ______r; })) __list_splice(list, head->prev, head); } static inline __attribute__((always_inline)) void list_splice_init(struct list_head *list, struct list_head *head) { if (__builtin_constant_p(((!list_empty(list)))) ? !!((!list_empty(list))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/list.h", .line = 321, }; ______r = !!((!list_empty(list))); ______f.miss_hit[______r]++; ______r; })) { __list_splice(list, head, head->next); INIT_LIST_HEAD(list); } } static inline __attribute__((always_inline)) void list_splice_tail_init(struct list_head *list, struct list_head *head) { if (__builtin_constant_p(((!list_empty(list)))) ? !!((!list_empty(list))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/list.h", .line = 338, }; ______r = !!((!list_empty(list))); ______f.miss_hit[______r]++; ______r; })) { __list_splice(list, head->prev, head); INIT_LIST_HEAD(list); } } static inline __attribute__((always_inline)) void INIT_HLIST_NODE(struct hlist_node *h) { h->next = ((void *)0); h->pprev = ((void *)0); } static inline __attribute__((always_inline)) int hlist_unhashed(const struct hlist_node *h) { return !h->pprev; } static inline __attribute__((always_inline)) int hlist_empty(const struct hlist_head *h) { return !h->first; } static inline __attribute__((always_inline)) void __hlist_del(struct hlist_node *n) { struct hlist_node *next = n->next; struct hlist_node **pprev = n->pprev; *pprev = next; if (__builtin_constant_p(((next))) ? !!((next)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/list.h", .line = 591, }; ______r = !!((next)); ______f.miss_hit[______r]++; ______r; })) next->pprev = pprev; } static inline __attribute__((always_inline)) void hlist_del(struct hlist_node *n) { __hlist_del(n); n->next = ((void *) 0x00100100 + (0x0UL)); n->pprev = ((void *) 0x00200200 + (0x0UL)); } static inline __attribute__((always_inline)) void hlist_del_init(struct hlist_node *n) { if (__builtin_constant_p(((!hlist_unhashed(n)))) ? !!((!hlist_unhashed(n))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/list.h", .line = 604, }; ______r = !!((!hlist_unhashed(n))); ______f.miss_hit[______r]++; ______r; })) { __hlist_del(n); INIT_HLIST_NODE(n); } } static inline __attribute__((always_inline)) void hlist_add_head(struct hlist_node *n, struct hlist_head *h) { struct hlist_node *first = h->first; n->next = first; if (__builtin_constant_p(((first))) ? !!((first)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/list.h", .line = 614, }; ______r = !!((first)); ______f.miss_hit[______r]++; ______r; })) first->pprev = &n->next; h->first = n; n->pprev = &h->first; } static inline __attribute__((always_inline)) void hlist_add_before(struct hlist_node *n, struct hlist_node *next) { n->pprev = next->pprev; n->next = next; next->pprev = &n->next; *(n->pprev) = n; } static inline __attribute__((always_inline)) void hlist_add_after(struct hlist_node *n, struct hlist_node *next) { next->next = n->next; n->next = next; next->pprev = &n->next; if (__builtin_constant_p(((next->next))) ? !!((next->next)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/list.h", .line = 637, }; ______r = !!((next->next)); ______f.miss_hit[______r]++; ______r; })) next->next->pprev = &next->next; } static inline __attribute__((always_inline)) void hlist_add_fake(struct hlist_node *n) { n->pprev = &n->next; } static inline __attribute__((always_inline)) void hlist_move_list(struct hlist_head *old, struct hlist_head *new) { new->first = old->first; if (__builtin_constant_p(((new->first))) ? !!((new->first)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/list.h", .line = 655, }; ______r = !!((new->first)); ______f.miss_hit[______r]++; ______r; })) new->first->pprev = &new->first; old->first = ((void *)0); } extern void add_preempt_count(int val); extern void sub_preempt_count(int val); __attribute__((regparm(0))) void preempt_schedule(void); struct preempt_notifier; struct preempt_ops { void (*sched_in)(struct preempt_notifier *notifier, int cpu); void (*sched_out)(struct preempt_notifier *notifier, struct task_struct *next); }; struct preempt_notifier { struct hlist_node link; struct preempt_ops *ops; }; void preempt_notifier_register(struct preempt_notifier *notifier); void preempt_notifier_unregister(struct preempt_notifier *notifier); static inline __attribute__((always_inline)) void preempt_notifier_init(struct preempt_notifier *notifier, struct preempt_ops *ops) { INIT_HLIST_NODE(¬ifier->link); notifier->ops = ops; } struct task_struct; struct lockdep_map; extern int prove_locking; extern int lock_stat; struct task_struct; extern int debug_locks; extern int debug_locks_silent; static inline __attribute__((always_inline)) int __debug_locks_off(void) { return ({ __typeof(*((&debug_locks))) __x = ((0)); switch (sizeof(*&debug_locks)) { case 1: { volatile u8 *__ptr = (volatile u8 *)((&debug_locks)); asm volatile("xchgb %0,%1" : "=q" (__x), "+m" (*__ptr) : "0" (__x) : "memory"); break; } case 2: { volatile u16 *__ptr = (volatile u16 *)((&debug_locks)); asm volatile("xchgw %0,%1" : "=r" (__x), "+m" (*__ptr) : "0" (__x) : "memory"); break; } case 4: { volatile u32 *__ptr = (volatile u32 *)((&debug_locks)); asm volatile("xchgl %0,%1" : "=r" (__x), "+m" (*__ptr) : "0" (__x) : "memory"); break; } default: __xchg_wrong_size(); } __x; }); } extern int debug_locks_off(void); struct task_struct; extern void debug_show_all_locks(void); extern void debug_show_held_locks(struct task_struct *task); extern void debug_check_no_locks_freed(const void *from, unsigned long len); extern void debug_check_no_locks_held(struct task_struct *task); struct task_struct; struct pt_regs; struct task_struct; struct stack_trace { unsigned int nr_entries, max_entries; unsigned long *entries; int skip; }; extern void save_stack_trace(struct stack_trace *trace); extern void save_stack_trace_regs(struct stack_trace *trace, struct pt_regs *regs); extern void save_stack_trace_tsk(struct task_struct *tsk, struct stack_trace *trace); extern void print_stack_trace(struct stack_trace *trace, int spaces); extern void save_stack_trace_user(struct stack_trace *trace); struct lockdep_subclass_key { char __one_byte; } __attribute__ ((__packed__)); struct lock_class_key { struct lockdep_subclass_key subkeys[8UL]; }; extern struct lock_class_key __lockdep_no_validate__; struct lock_class { struct list_head hash_entry; struct list_head lock_entry; struct lockdep_subclass_key *key; unsigned int subclass; unsigned int dep_gen_id; unsigned long usage_mask; struct stack_trace usage_traces[(1+3*4)]; struct list_head locks_after, locks_before; unsigned int version; unsigned long ops; const char *name; int name_version; }; struct lockdep_map { struct lock_class_key *key; struct lock_class *class_cache[2]; const char *name; }; struct lock_list { struct list_head entry; struct lock_class *class; struct stack_trace trace; int distance; struct lock_list *parent; }; struct lock_chain { u8 irq_context; u8 depth; u16 base; struct list_head entry; u64 chain_key; }; struct held_lock { u64 prev_chain_key; unsigned long acquire_ip; struct lockdep_map *instance; struct lockdep_map *nest_lock; unsigned int class_idx:13; unsigned int irq_context:2; unsigned int trylock:1; unsigned int read:2; unsigned int check:2; unsigned int hardirqs_off:1; unsigned int references:11; }; extern void lockdep_init(void); extern void lockdep_info(void); extern void lockdep_reset(void); extern void lockdep_reset_lock(struct lockdep_map *lock); extern void lockdep_free_key_range(void *start, unsigned long size); extern void lockdep_sys_exit(void); extern void lockdep_off(void); extern void lockdep_on(void); extern void lockdep_init_map(struct lockdep_map *lock, const char *name, struct lock_class_key *key, int subclass); static inline __attribute__((always_inline)) int lockdep_match_key(struct lockdep_map *lock, struct lock_class_key *key) { return lock->key == key; } extern void lock_acquire(struct lockdep_map *lock, unsigned int subclass, int trylock, int read, int check, struct lockdep_map *nest_lock, unsigned long ip); extern void lock_release(struct lockdep_map *lock, int nested, unsigned long ip); extern int lock_is_held(struct lockdep_map *lock); extern void lock_set_class(struct lockdep_map *lock, const char *name, struct lock_class_key *key, unsigned int subclass, unsigned long ip); static inline __attribute__((always_inline)) void lock_set_subclass(struct lockdep_map *lock, unsigned int subclass, unsigned long ip) { lock_set_class(lock, lock->name, lock->key, subclass, ip); } extern void lockdep_set_current_reclaim_state(gfp_t gfp_mask); extern void lockdep_clear_current_reclaim_state(void); extern void lockdep_trace_alloc(gfp_t mask); extern void print_irqtrace_events(struct task_struct *curr); extern void ftrace_nmi_enter(void); extern void ftrace_nmi_exit(void); extern void cpu_idle(void); typedef void (*smp_call_func_t)(void *info); struct call_single_data { struct list_head list; smp_call_func_t func; void *info; u16 flags; u16 priv; }; extern unsigned int total_cpus; int smp_call_function_single(int cpuid, smp_call_func_t func, void *info, int wait); struct mpf_intel { char signature[4]; unsigned int physptr; unsigned char length; unsigned char specification; unsigned char checksum; unsigned char feature1; unsigned char feature2; unsigned char feature3; unsigned char feature4; unsigned char feature5; }; struct mpc_table { char signature[4]; unsigned short length; char spec; char checksum; char oem[8]; char productid[12]; unsigned int oemptr; unsigned short oemsize; unsigned short oemcount; unsigned int lapic; unsigned int reserved; }; struct mpc_cpu { unsigned char type; unsigned char apicid; unsigned char apicver; unsigned char cpuflag; unsigned int cpufeature; unsigned int featureflag; unsigned int reserved[2]; }; struct mpc_bus { unsigned char type; unsigned char busid; unsigned char bustype[6]; }; struct mpc_ioapic { unsigned char type; unsigned char apicid; unsigned char apicver; unsigned char flags; unsigned int apicaddr; }; struct mpc_intsrc { unsigned char type; unsigned char irqtype; unsigned short irqflag; unsigned char srcbus; unsigned char srcbusirq; unsigned char dstapic; unsigned char dstirq; }; enum mp_irq_source_types { mp_INT = 0, mp_NMI = 1, mp_SMI = 2, mp_ExtINT = 3 }; struct mpc_lintsrc { unsigned char type; unsigned char irqtype; unsigned short irqflag; unsigned char srcbusid; unsigned char srcbusirq; unsigned char destapic; unsigned char destapiclint; }; struct mpc_oemtable { char signature[4]; unsigned short length; char rev; char checksum; char mpc[8]; }; enum mp_bustype { MP_BUS_ISA = 1, MP_BUS_EISA, MP_BUS_PCI, MP_BUS_MCA, }; struct screen_info { __u8 orig_x; __u8 orig_y; __u16 ext_mem_k; __u16 orig_video_page; __u8 orig_video_mode; __u8 orig_video_cols; __u8 flags; __u8 unused2; __u16 orig_video_ega_bx; __u16 unused3; __u8 orig_video_lines; __u8 orig_video_isVGA; __u16 orig_video_points; __u16 lfb_width; __u16 lfb_height; __u16 lfb_depth; __u32 lfb_base; __u32 lfb_size; __u16 cl_magic, cl_offset; __u16 lfb_linelength; __u8 red_size; __u8 red_pos; __u8 green_size; __u8 green_pos; __u8 blue_size; __u8 blue_pos; __u8 rsvd_size; __u8 rsvd_pos; __u16 vesapm_seg; __u16 vesapm_off; __u16 pages; __u16 vesa_attributes; __u32 capabilities; __u8 _reserved[6]; } __attribute__((packed)); extern struct screen_info screen_info; typedef unsigned short apm_event_t; typedef unsigned short apm_eventinfo_t; struct apm_bios_info { __u16 version; __u16 cseg; __u32 offset; __u16 cseg_16; __u16 dseg; __u16 flags; __u16 cseg_len; __u16 cseg_16_len; __u16 dseg_len; }; struct apm_info { struct apm_bios_info bios; unsigned short connection_version; int get_power_status_broken; int get_power_status_swabinminutes; int allow_ints; int forbid_idle; int realmode_power_off; int disabled; }; extern struct apm_info apm_info; struct edd_device_params { __u16 length; __u16 info_flags; __u32 num_default_cylinders; __u32 num_default_heads; __u32 sectors_per_track; __u64 number_of_sectors; __u16 bytes_per_sector; __u32 dpte_ptr; __u16 key; __u8 device_path_info_length; __u8 reserved2; __u16 reserved3; __u8 host_bus_type[4]; __u8 interface_type[8]; union { struct { __u16 base_address; __u16 reserved1; __u32 reserved2; } __attribute__ ((packed)) isa; struct { __u8 bus; __u8 slot; __u8 function; __u8 channel; __u32 reserved; } __attribute__ ((packed)) pci; struct { __u64 reserved; } __attribute__ ((packed)) ibnd; struct { __u64 reserved; } __attribute__ ((packed)) xprs; struct { __u64 reserved; } __attribute__ ((packed)) htpt; struct { __u64 reserved; } __attribute__ ((packed)) unknown; } interface_path; union { struct { __u8 device; __u8 reserved1; __u16 reserved2; __u32 reserved3; __u64 reserved4; } __attribute__ ((packed)) ata; struct { __u8 device; __u8 lun; __u8 reserved1; __u8 reserved2; __u32 reserved3; __u64 reserved4; } __attribute__ ((packed)) atapi; struct { __u16 id; __u64 lun; __u16 reserved1; __u32 reserved2; } __attribute__ ((packed)) scsi; struct { __u64 serial_number; __u64 reserved; } __attribute__ ((packed)) usb; struct { __u64 eui; __u64 reserved; } __attribute__ ((packed)) i1394; struct { __u64 wwid; __u64 lun; } __attribute__ ((packed)) fibre; struct { __u64 identity_tag; __u64 reserved; } __attribute__ ((packed)) i2o; struct { __u32 array_number; __u32 reserved1; __u64 reserved2; } __attribute__ ((packed)) raid; struct { __u8 device; __u8 reserved1; __u16 reserved2; __u32 reserved3; __u64 reserved4; } __attribute__ ((packed)) sata; struct { __u64 reserved1; __u64 reserved2; } __attribute__ ((packed)) unknown; } device_path; __u8 reserved4; __u8 checksum; } __attribute__ ((packed)); struct edd_info { __u8 device; __u8 version; __u16 interface_support; __u16 legacy_max_cylinder; __u8 legacy_max_head; __u8 legacy_sectors_per_track; struct edd_device_params params; } __attribute__ ((packed)); struct edd { unsigned int mbr_signature[16]; struct edd_info edd_info[6]; unsigned char mbr_signature_nr; unsigned char edd_info_nr; }; extern struct edd edd; struct e820entry { __u64 addr; __u64 size; __u32 type; } __attribute__((packed)); struct e820map { __u32 nr_map; struct e820entry map[128]; }; extern struct e820map e820; extern struct e820map e820_saved; extern unsigned long pci_mem_start; extern int e820_any_mapped(u64 start, u64 end, unsigned type); extern int e820_all_mapped(u64 start, u64 end, unsigned type); extern void e820_add_region(u64 start, u64 size, int type); extern void e820_print_map(char *who); extern int sanitize_e820_map(struct e820entry *biosmap, int max_nr_map, u32 *pnr_map); extern u64 e820_update_range(u64 start, u64 size, unsigned old_type, unsigned new_type); extern u64 e820_remove_range(u64 start, u64 size, unsigned old_type, int checktype); extern void update_e820(void); extern void e820_setup_gap(void); extern int e820_search_gap(unsigned long *gapstart, unsigned long *gapsize, unsigned long start_addr, unsigned long long end_addr); struct setup_data; extern void parse_e820_ext(struct setup_data *data); extern void e820_mark_nosave_regions(unsigned long limit_pfn); static inline __attribute__((always_inline)) void early_memtest(unsigned long start, unsigned long end) { } extern unsigned long e820_end_of_ram_pfn(void); extern unsigned long e820_end_of_low_ram_pfn(void); extern u64 early_reserve_e820(u64 startt, u64 sizet, u64 align); void memblock_x86_fill(void); void memblock_find_dma_reserve(void); extern void finish_e820_parsing(void); extern void e820_reserve_resources(void); extern void e820_reserve_resources_late(void); extern void setup_memory_map(void); extern char *default_machine_specific_memory_setup(void); static inline __attribute__((always_inline)) bool is_ISA_range(u64 s, u64 e) { return s >= 0xa0000 && e <= 0x100000; } struct resource { resource_size_t start; resource_size_t end; const char *name; unsigned long flags; struct resource *parent, *sibling, *child; }; struct resource_list { struct resource_list *next; struct resource *res; struct pci_dev *dev; }; extern struct resource ioport_resource; extern struct resource iomem_resource; extern struct resource *request_resource_conflict(struct resource *root, struct resource *new); extern int request_resource(struct resource *root, struct resource *new); extern int release_resource(struct resource *new); void release_child_resources(struct resource *new); extern void reserve_region_with_split(struct resource *root, resource_size_t start, resource_size_t end, const char *name); extern struct resource *insert_resource_conflict(struct resource *parent, struct resource *new); extern int insert_resource(struct resource *parent, struct resource *new); extern void insert_resource_expand_to_fit(struct resource *root, struct resource *new); extern void arch_remove_reservations(struct resource *avail); extern int allocate_resource(struct resource *root, struct resource *new, resource_size_t size, resource_size_t min, resource_size_t max, resource_size_t align, resource_size_t (*alignf)(void *, const struct resource *, resource_size_t, resource_size_t), void *alignf_data); int adjust_resource(struct resource *res, resource_size_t start, resource_size_t size); resource_size_t resource_alignment(struct resource *res); static inline __attribute__((always_inline)) resource_size_t resource_size(const struct resource *res) { return res->end - res->start + 1; } static inline __attribute__((always_inline)) unsigned long resource_type(const struct resource *res) { return res->flags & 0x00001f00; } extern struct resource * __request_region(struct resource *, resource_size_t start, resource_size_t n, const char *name, int flags); extern int __check_region(struct resource *, resource_size_t, resource_size_t); extern void __release_region(struct resource *, resource_size_t, resource_size_t); static inline __attribute__((always_inline)) int __attribute__((deprecated)) check_region(resource_size_t s, resource_size_t n) { return __check_region(&ioport_resource, s, n); } struct device; extern struct resource * __devm_request_region(struct device *dev, struct resource *parent, resource_size_t start, resource_size_t n, const char *name); extern void __devm_release_region(struct device *dev, struct resource *parent, resource_size_t start, resource_size_t n); extern int iomem_map_sanity_check(resource_size_t addr, unsigned long size); extern int iomem_is_exclusive(u64 addr); extern int walk_system_ram_range(unsigned long start_pfn, unsigned long nr_pages, void *arg, int (*func)(unsigned long, unsigned long, void *)); struct ist_info { __u32 signature; __u32 command; __u32 event; __u32 perf_level; }; extern struct ist_info ist_info; struct edid_info { unsigned char dummy[128]; }; extern struct edid_info edid_info; struct setup_data { __u64 next; __u32 type; __u32 len; __u8 data[0]; }; struct setup_header { __u8 setup_sects; __u16 root_flags; __u32 syssize; __u16 ram_size; __u16 vid_mode; __u16 root_dev; __u16 boot_flag; __u16 jump; __u32 header; __u16 version; __u32 realmode_swtch; __u16 start_sys; __u16 kernel_version; __u8 type_of_loader; __u8 loadflags; __u16 setup_move_size; __u32 code32_start; __u32 ramdisk_image; __u32 ramdisk_size; __u32 bootsect_kludge; __u16 heap_end_ptr; __u8 ext_loader_ver; __u8 ext_loader_type; __u32 cmd_line_ptr; __u32 initrd_addr_max; __u32 kernel_alignment; __u8 relocatable_kernel; __u8 _pad2[3]; __u32 cmdline_size; __u32 hardware_subarch; __u64 hardware_subarch_data; __u32 payload_offset; __u32 payload_length; __u64 setup_data; } __attribute__((packed)); struct sys_desc_table { __u16 length; __u8 table[14]; }; struct olpc_ofw_header { __u32 ofw_magic; __u32 ofw_version; __u32 cif_handler; __u32 irq_desc_table; } __attribute__((packed)); struct efi_info { __u32 efi_loader_signature; __u32 efi_systab; __u32 efi_memdesc_size; __u32 efi_memdesc_version; __u32 efi_memmap; __u32 efi_memmap_size; __u32 efi_systab_hi; __u32 efi_memmap_hi; }; struct boot_params { struct screen_info screen_info; struct apm_bios_info apm_bios_info; __u8 _pad2[4]; __u64 tboot_addr; struct ist_info ist_info; __u8 _pad3[16]; __u8 hd0_info[16]; __u8 hd1_info[16]; struct sys_desc_table sys_desc_table; struct olpc_ofw_header olpc_ofw_header; __u8 _pad4[128]; struct edid_info edid_info; struct efi_info efi_info; __u32 alt_mem_k; __u32 scratch; __u8 e820_entries; __u8 eddbuf_entries; __u8 edd_mbr_sig_buf_entries; __u8 _pad6[6]; struct setup_header hdr; __u8 _pad7[0x290-0x1f1-sizeof(struct setup_header)]; __u32 edd_mbr_sig_buffer[16]; struct e820entry e820_map[128]; __u8 _pad8[48]; struct edd_info eddbuf[6]; __u8 _pad9[276]; } __attribute__((packed)); enum { X86_SUBARCH_PC = 0, X86_SUBARCH_LGUEST, X86_SUBARCH_XEN, X86_SUBARCH_MRST, X86_SUBARCH_CE4100, X86_NR_SUBARCHS, }; struct mpc_bus; struct mpc_cpu; struct mpc_table; struct x86_init_mpparse { void (*mpc_record)(unsigned int mode); void (*setup_ioapic_ids)(void); int (*mpc_apic_id)(struct mpc_cpu *m); void (*smp_read_mpc_oem)(struct mpc_table *mpc); void (*mpc_oem_pci_bus)(struct mpc_bus *m); void (*mpc_oem_bus_info)(struct mpc_bus *m, char *name); void (*find_smp_config)(void); void (*get_smp_config)(unsigned int early); }; struct x86_init_resources { void (*probe_roms)(void); void (*reserve_resources)(void); char *(*memory_setup)(void); }; struct x86_init_irqs { void (*pre_vector_init)(void); void (*intr_init)(void); void (*trap_init)(void); }; struct x86_init_oem { void (*arch_setup)(void); void (*banner)(void); }; struct x86_init_mapping { void (*pagetable_reserve)(u64 start, u64 end); }; struct x86_init_paging { void (*pagetable_setup_start)(pgd_t *base); void (*pagetable_setup_done)(pgd_t *base); }; struct x86_init_timers { void (*setup_percpu_clockev)(void); void (*tsc_pre_init)(void); void (*timer_init)(void); void (*wallclock_init)(void); }; struct x86_init_iommu { int (*iommu_init)(void); }; struct x86_init_pci { int (*arch_init)(void); int (*init)(void); void (*init_irq)(void); void (*fixup_irqs)(void); }; struct x86_init_ops { struct x86_init_resources resources; struct x86_init_mpparse mpparse; struct x86_init_irqs irqs; struct x86_init_oem oem; struct x86_init_mapping mapping; struct x86_init_paging paging; struct x86_init_timers timers; struct x86_init_iommu iommu; struct x86_init_pci pci; }; struct x86_cpuinit_ops { void (*setup_percpu_clockev)(void); }; struct x86_platform_ops { unsigned long (*calibrate_tsc)(void); unsigned long (*get_wallclock)(void); int (*set_wallclock)(unsigned long nowtime); void (*iommu_shutdown)(void); bool (*is_untracked_pat_range)(u64 start, u64 end); void (*nmi_init)(void); int (*i8042_detect)(void); }; struct pci_dev; struct x86_msi_ops { int (*setup_msi_irqs)(struct pci_dev *dev, int nvec, int type); void (*teardown_msi_irq)(unsigned int irq); void (*teardown_msi_irqs)(struct pci_dev *dev); }; extern struct x86_init_ops x86_init; extern struct x86_cpuinit_ops x86_cpuinit; extern struct x86_platform_ops x86_platform; extern struct x86_msi_ops x86_msi; extern void x86_init_noop(void); extern void x86_init_uint_noop(unsigned int unused); struct local_apic { struct { unsigned int __reserved[4]; } __reserved_01; struct { unsigned int __reserved[4]; } __reserved_02; struct { unsigned int __reserved_1 : 24, phys_apic_id : 4, __reserved_2 : 4; unsigned int __reserved[3]; } id; const struct { unsigned int version : 8, __reserved_1 : 8, max_lvt : 8, __reserved_2 : 8; unsigned int __reserved[3]; } version; struct { unsigned int __reserved[4]; } __reserved_03; struct { unsigned int __reserved[4]; } __reserved_04; struct { unsigned int __reserved[4]; } __reserved_05; struct { unsigned int __reserved[4]; } __reserved_06; struct { unsigned int priority : 8, __reserved_1 : 24; unsigned int __reserved_2[3]; } tpr; const struct { unsigned int priority : 8, __reserved_1 : 24; unsigned int __reserved_2[3]; } apr; const struct { unsigned int priority : 8, __reserved_1 : 24; unsigned int __reserved_2[3]; } ppr; struct { unsigned int eoi; unsigned int __reserved[3]; } eoi; struct { unsigned int __reserved[4]; } __reserved_07; struct { unsigned int __reserved_1 : 24, logical_dest : 8; unsigned int __reserved_2[3]; } ldr; struct { unsigned int __reserved_1 : 28, model : 4; unsigned int __reserved_2[3]; } dfr; struct { unsigned int spurious_vector : 8, apic_enabled : 1, focus_cpu : 1, __reserved_2 : 22; unsigned int __reserved_3[3]; } svr; struct { unsigned int bitfield; unsigned int __reserved[3]; } isr [8]; struct { unsigned int bitfield; unsigned int __reserved[3]; } tmr [8]; struct { unsigned int bitfield; unsigned int __reserved[3]; } irr [8]; union { struct { unsigned int send_cs_error : 1, receive_cs_error : 1, send_accept_error : 1, receive_accept_error : 1, __reserved_1 : 1, send_illegal_vector : 1, receive_illegal_vector : 1, illegal_register_address : 1, __reserved_2 : 24; unsigned int __reserved_3[3]; } error_bits; struct { unsigned int errors; unsigned int __reserved_3[3]; } all_errors; } esr; struct { unsigned int __reserved[4]; } __reserved_08; struct { unsigned int __reserved[4]; } __reserved_09; struct { unsigned int __reserved[4]; } __reserved_10; struct { unsigned int __reserved[4]; } __reserved_11; struct { unsigned int __reserved[4]; } __reserved_12; struct { unsigned int __reserved[4]; } __reserved_13; struct { unsigned int __reserved[4]; } __reserved_14; struct { unsigned int vector : 8, delivery_mode : 3, destination_mode : 1, delivery_status : 1, __reserved_1 : 1, level : 1, trigger : 1, __reserved_2 : 2, shorthand : 2, __reserved_3 : 12; unsigned int __reserved_4[3]; } icr1; struct { union { unsigned int __reserved_1 : 24, phys_dest : 4, __reserved_2 : 4; unsigned int __reserved_3 : 24, logical_dest : 8; } dest; unsigned int __reserved_4[3]; } icr2; struct { unsigned int vector : 8, __reserved_1 : 4, delivery_status : 1, __reserved_2 : 3, mask : 1, timer_mode : 1, __reserved_3 : 14; unsigned int __reserved_4[3]; } lvt_timer; struct { unsigned int vector : 8, delivery_mode : 3, __reserved_1 : 1, delivery_status : 1, __reserved_2 : 3, mask : 1, __reserved_3 : 15; unsigned int __reserved_4[3]; } lvt_thermal; struct { unsigned int vector : 8, delivery_mode : 3, __reserved_1 : 1, delivery_status : 1, __reserved_2 : 3, mask : 1, __reserved_3 : 15; unsigned int __reserved_4[3]; } lvt_pc; struct { unsigned int vector : 8, delivery_mode : 3, __reserved_1 : 1, delivery_status : 1, polarity : 1, remote_irr : 1, trigger : 1, mask : 1, __reserved_2 : 15; unsigned int __reserved_3[3]; } lvt_lint0; struct { unsigned int vector : 8, delivery_mode : 3, __reserved_1 : 1, delivery_status : 1, polarity : 1, remote_irr : 1, trigger : 1, mask : 1, __reserved_2 : 15; unsigned int __reserved_3[3]; } lvt_lint1; struct { unsigned int vector : 8, __reserved_1 : 4, delivery_status : 1, __reserved_2 : 3, mask : 1, __reserved_3 : 15; unsigned int __reserved_4[3]; } lvt_error; struct { unsigned int initial_count; unsigned int __reserved_2[3]; } timer_icr; const struct { unsigned int curr_count; unsigned int __reserved_2[3]; } timer_ccr; struct { unsigned int __reserved[4]; } __reserved_16; struct { unsigned int __reserved[4]; } __reserved_17; struct { unsigned int __reserved[4]; } __reserved_18; struct { unsigned int __reserved[4]; } __reserved_19; struct { unsigned int divisor : 4, __reserved_1 : 28; unsigned int __reserved_2[3]; } timer_dcr; struct { unsigned int __reserved[4]; } __reserved_20; } __attribute__ ((packed)); enum ioapic_irq_destination_types { dest_Fixed = 0, dest_LowestPrio = 1, dest_SMI = 2, dest__reserved_1 = 3, dest_NMI = 4, dest_INIT = 5, dest__reserved_2 = 6, dest_ExtINT = 7 }; extern int apic_version[]; extern int pic_mode; extern unsigned int def_to_bigsmp; extern unsigned long mp_bus_not_pci[(((260) + (8 * sizeof(long)) - 1) / (8 * sizeof(long)))]; extern unsigned int boot_cpu_physical_apicid; extern unsigned int max_physical_apicid; extern int mpc_default_type; extern unsigned long mp_lapic_addr; extern int smp_found_config; static inline __attribute__((always_inline)) void get_smp_config(void) { x86_init.mpparse.get_smp_config(0); } static inline __attribute__((always_inline)) void early_get_smp_config(void) { x86_init.mpparse.get_smp_config(1); } static inline __attribute__((always_inline)) void find_smp_config(void) { x86_init.mpparse.find_smp_config(); } extern void early_reserve_e820_mpc_new(void); extern int enable_update_mptable; extern int default_mpc_apic_id(struct mpc_cpu *m); extern void default_smp_read_mpc_oem(struct mpc_table *mpc); extern void default_mpc_oem_bus_info(struct mpc_bus *m, char *str); extern void default_find_smp_config(void); extern void default_get_smp_config(unsigned int early); void __attribute__ ((__section__(".cpuinit.text"))) __attribute__((__cold__)) __attribute__((no_instrument_function)) generic_processor_info(int apicid, int version); extern void mp_register_ioapic(int id, u32 address, u32 gsi_base); extern void mp_override_legacy_irq(u8 bus_irq, u8 polarity, u8 trigger, u32 gsi); extern void mp_config_acpi_legacy_irqs(void); struct device; extern int mp_register_gsi(struct device *dev, u32 gsi, int edge_level, int active_high_low); struct physid_mask { unsigned long mask[(((256) + (8 * sizeof(long)) - 1) / (8 * sizeof(long)))]; }; typedef struct physid_mask physid_mask_t; static inline __attribute__((always_inline)) unsigned long physids_coerce(physid_mask_t *map) { return map->mask[0]; } static inline __attribute__((always_inline)) void physids_promote(unsigned long physids, physid_mask_t *map) { bitmap_zero((*map).mask, 256); map->mask[0] = physids; } static inline __attribute__((always_inline)) void physid_set_mask_of_physid(int physid, physid_mask_t *map) { bitmap_zero((*map).mask, 256); set_bit(physid, (*map).mask); } extern physid_mask_t phys_cpu_present_map; extern int generic_mps_oem_check(struct mpc_table *, char *, char *); extern int default_acpi_madt_oem_check(char *, char *); extern void local_bh_disable(void); extern void _local_bh_enable(void); extern void local_bh_enable(void); extern void local_bh_enable_ip(unsigned long ip); typedef struct arch_spinlock { unsigned int slock; } arch_spinlock_t; typedef struct { unsigned int lock; } arch_rwlock_t; typedef struct raw_spinlock { arch_spinlock_t raw_lock; unsigned int magic, owner_cpu; void *owner; struct lockdep_map dep_map; } raw_spinlock_t; typedef struct spinlock { union { struct raw_spinlock rlock; struct { u8 __padding[(__builtin_offsetof(struct raw_spinlock,dep_map))]; struct lockdep_map dep_map; }; }; } spinlock_t; typedef struct { arch_rwlock_t raw_lock; unsigned int magic, owner_cpu; void *owner; struct lockdep_map dep_map; } rwlock_t; static inline __attribute__((always_inline)) __attribute__((always_inline)) void __ticket_spin_lock(arch_spinlock_t *lock) { short inc = 0x0100; asm volatile ( ".section .smp_locks,\"a\"\n" ".balign 4\n" ".long 671f - .\n" ".previous\n" "671:" "\n\tlock; " "xaddw %w0, %1\n" "1:\t" "cmpb %h0, %b0\n\t" "je 2f\n\t" "rep ; nop\n\t" "movb %1, %b0\n\t" "jmp 1b\n" "2:" : "+Q" (inc), "+m" (lock->slock) : : "memory", "cc"); } static inline __attribute__((always_inline)) __attribute__((always_inline)) int __ticket_spin_trylock(arch_spinlock_t *lock) { int tmp, new; asm volatile("movzwl %2, %0\n\t" "cmpb %h0,%b0\n\t" "leal 0x100(%" "k" "0), %1\n\t" "jne 1f\n\t" ".section .smp_locks,\"a\"\n" ".balign 4\n" ".long 671f - .\n" ".previous\n" "671:" "\n\tlock; " "cmpxchgw %w1,%2\n\t" "1:" "sete %b1\n\t" "movzbl %b1,%0\n\t" : "=&a" (tmp), "=&q" (new), "+m" (lock->slock) : : "memory", "cc"); return tmp; } static inline __attribute__((always_inline)) __attribute__((always_inline)) void __ticket_spin_unlock(arch_spinlock_t *lock) { asm volatile( "incb %0" : "+m" (lock->slock) : : "memory", "cc"); } static inline __attribute__((always_inline)) int __ticket_spin_is_locked(arch_spinlock_t *lock) { int tmp = (*(volatile typeof(lock->slock) *)&(lock->slock)); return !!(((tmp >> 8) ^ tmp) & ((1 << 8) - 1)); } static inline __attribute__((always_inline)) int __ticket_spin_is_contended(arch_spinlock_t *lock) { int tmp = (*(volatile typeof(lock->slock) *)&(lock->slock)); return (((tmp >> 8) - tmp) & ((1 << 8) - 1)) > 1; } static inline __attribute__((always_inline)) int arch_spin_is_locked(arch_spinlock_t *lock) { return __ticket_spin_is_locked(lock); } static inline __attribute__((always_inline)) int arch_spin_is_contended(arch_spinlock_t *lock) { return __ticket_spin_is_contended(lock); } static inline __attribute__((always_inline)) __attribute__((always_inline)) void arch_spin_lock(arch_spinlock_t *lock) { __ticket_spin_lock(lock); } static inline __attribute__((always_inline)) __attribute__((always_inline)) int arch_spin_trylock(arch_spinlock_t *lock) { return __ticket_spin_trylock(lock); } static inline __attribute__((always_inline)) __attribute__((always_inline)) void arch_spin_unlock(arch_spinlock_t *lock) { __ticket_spin_unlock(lock); } static inline __attribute__((always_inline)) __attribute__((always_inline)) void arch_spin_lock_flags(arch_spinlock_t *lock, unsigned long flags) { arch_spin_lock(lock); } static inline __attribute__((always_inline)) void arch_spin_unlock_wait(arch_spinlock_t *lock) { while (arch_spin_is_locked(lock)) cpu_relax(); } static inline __attribute__((always_inline)) int arch_read_can_lock(arch_rwlock_t *lock) { return (int)(lock)->lock > 0; } static inline __attribute__((always_inline)) int arch_write_can_lock(arch_rwlock_t *lock) { return (lock)->lock == 0x01000000; } static inline __attribute__((always_inline)) void arch_read_lock(arch_rwlock_t *rw) { asm volatile(".section .smp_locks,\"a\"\n" ".balign 4\n" ".long 671f - .\n" ".previous\n" "671:" "\n\tlock; " " subl $1,(%0)\n\t" "jns 1f\n" "call __read_lock_failed\n\t" "1:\n" ::"a" (rw) : "memory"); } static inline __attribute__((always_inline)) void arch_write_lock(arch_rwlock_t *rw) { asm volatile(".section .smp_locks,\"a\"\n" ".balign 4\n" ".long 671f - .\n" ".previous\n" "671:" "\n\tlock; " " subl %1,(%0)\n\t" "jz 1f\n" "call __write_lock_failed\n\t" "1:\n" ::"a" (rw), "i" (0x01000000) : "memory"); } static inline __attribute__((always_inline)) int arch_read_trylock(arch_rwlock_t *lock) { atomic_t *count = (atomic_t *)lock; if (__builtin_constant_p((((atomic_sub_return(1, count)) >= 0))) ? !!(((atomic_sub_return(1, count)) >= 0)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/spinlock.h", .line = 271, }; ______r = !!(((atomic_sub_return(1, count)) >= 0)); ______f.miss_hit[______r]++; ______r; })) return 1; atomic_inc(count); return 0; } static inline __attribute__((always_inline)) int arch_write_trylock(arch_rwlock_t *lock) { atomic_t *count = (atomic_t *)lock; if (__builtin_constant_p(((atomic_sub_and_test(0x01000000, count)))) ? !!((atomic_sub_and_test(0x01000000, count))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/spinlock.h", .line = 281, }; ______r = !!((atomic_sub_and_test(0x01000000, count))); ______f.miss_hit[______r]++; ______r; })) return 1; atomic_add(0x01000000, count); return 0; } static inline __attribute__((always_inline)) void arch_read_unlock(arch_rwlock_t *rw) { asm volatile(".section .smp_locks,\"a\"\n" ".balign 4\n" ".long 671f - .\n" ".previous\n" "671:" "\n\tlock; " "incl %0" :"+m" (rw->lock) : : "memory"); } static inline __attribute__((always_inline)) void arch_write_unlock(arch_rwlock_t *rw) { asm volatile(".section .smp_locks,\"a\"\n" ".balign 4\n" ".long 671f - .\n" ".previous\n" "671:" "\n\tlock; " "addl %1, %0" : "+m" (rw->lock) : "i" (0x01000000) : "memory"); } static inline __attribute__((always_inline)) void smp_mb__after_lock(void) { } extern void __raw_spin_lock_init(raw_spinlock_t *lock, const char *name, struct lock_class_key *key); extern void do_raw_spin_lock(raw_spinlock_t *lock) ; extern int do_raw_spin_trylock(raw_spinlock_t *lock); extern void do_raw_spin_unlock(raw_spinlock_t *lock) ; extern void __rwlock_init(rwlock_t *lock, const char *name, struct lock_class_key *key); extern void do_raw_read_lock(rwlock_t *lock) ; extern int do_raw_read_trylock(rwlock_t *lock); extern void do_raw_read_unlock(rwlock_t *lock) ; extern void do_raw_write_lock(rwlock_t *lock) ; extern int do_raw_write_trylock(rwlock_t *lock); extern void do_raw_write_unlock(rwlock_t *lock) ; int in_lock_functions(unsigned long addr); void __attribute__((section(".spinlock.text"))) _raw_spin_lock(raw_spinlock_t *lock) ; void __attribute__((section(".spinlock.text"))) _raw_spin_lock_nested(raw_spinlock_t *lock, int subclass) ; void __attribute__((section(".spinlock.text"))) _raw_spin_lock_nest_lock(raw_spinlock_t *lock, struct lockdep_map *map) ; void __attribute__((section(".spinlock.text"))) _raw_spin_lock_bh(raw_spinlock_t *lock) ; void __attribute__((section(".spinlock.text"))) _raw_spin_lock_irq(raw_spinlock_t *lock) ; unsigned long __attribute__((section(".spinlock.text"))) _raw_spin_lock_irqsave(raw_spinlock_t *lock) ; unsigned long __attribute__((section(".spinlock.text"))) _raw_spin_lock_irqsave_nested(raw_spinlock_t *lock, int subclass) ; int __attribute__((section(".spinlock.text"))) _raw_spin_trylock(raw_spinlock_t *lock); int __attribute__((section(".spinlock.text"))) _raw_spin_trylock_bh(raw_spinlock_t *lock); void __attribute__((section(".spinlock.text"))) _raw_spin_unlock(raw_spinlock_t *lock) ; void __attribute__((section(".spinlock.text"))) _raw_spin_unlock_bh(raw_spinlock_t *lock) ; void __attribute__((section(".spinlock.text"))) _raw_spin_unlock_irq(raw_spinlock_t *lock) ; void __attribute__((section(".spinlock.text"))) _raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags) ; static inline __attribute__((always_inline)) int __raw_spin_trylock(raw_spinlock_t *lock) { do { add_preempt_count(1); __asm__ __volatile__("": : :"memory"); } while (0); if (__builtin_constant_p(((do_raw_spin_trylock(lock)))) ? !!((do_raw_spin_trylock(lock))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/spinlock_api_smp.h", .line = 89, }; ______r = !!((do_raw_spin_trylock(lock))); ______f.miss_hit[______r]++; ______r; })) { lock_acquire(&lock->dep_map, 0, 1, 0, 2, ((void *)0), (unsigned long)__builtin_return_address(0)); return 1; } do { do { __asm__ __volatile__("": : :"memory"); sub_preempt_count(1); } while (0); __asm__ __volatile__("": : :"memory"); do { if (__builtin_constant_p((((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/spinlock_api_smp.h", .line = 93, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/spinlock_api_smp.h", .line = 93, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/spinlock_api_smp.h", .line = 93, }; ______r = !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/spinlock_api_smp.h", .line = 93, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) preempt_schedule(); } while (0); } while (0); return 0; } static inline __attribute__((always_inline)) unsigned long __raw_spin_lock_irqsave(raw_spinlock_t *lock) { unsigned long flags; do { do { ({ unsigned long __dummy; typeof(flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); flags = arch_local_irq_save(); } while (0); trace_hardirqs_off(); } while (0); do { add_preempt_count(1); __asm__ __volatile__("": : :"memory"); } while (0); lock_acquire(&lock->dep_map, 0, 0, 0, 2, ((void *)0), (unsigned long)__builtin_return_address(0)); do_raw_spin_lock(lock); return flags; } static inline __attribute__((always_inline)) void __raw_spin_lock_irq(raw_spinlock_t *lock) { do { arch_local_irq_disable(); trace_hardirqs_off(); } while (0); do { add_preempt_count(1); __asm__ __volatile__("": : :"memory"); } while (0); lock_acquire(&lock->dep_map, 0, 0, 0, 2, ((void *)0), (unsigned long)__builtin_return_address(0)); do_raw_spin_lock(lock); } static inline __attribute__((always_inline)) void __raw_spin_lock_bh(raw_spinlock_t *lock) { local_bh_disable(); do { add_preempt_count(1); __asm__ __volatile__("": : :"memory"); } while (0); lock_acquire(&lock->dep_map, 0, 0, 0, 2, ((void *)0), (unsigned long)__builtin_return_address(0)); do_raw_spin_lock(lock); } static inline __attribute__((always_inline)) void __raw_spin_lock(raw_spinlock_t *lock) { do { add_preempt_count(1); __asm__ __volatile__("": : :"memory"); } while (0); lock_acquire(&lock->dep_map, 0, 0, 0, 2, ((void *)0), (unsigned long)__builtin_return_address(0)); do_raw_spin_lock(lock); } static inline __attribute__((always_inline)) void __raw_spin_unlock(raw_spinlock_t *lock) { lock_release(&lock->dep_map, 1, (unsigned long)__builtin_return_address(0)); do_raw_spin_unlock(lock); do { do { __asm__ __volatile__("": : :"memory"); sub_preempt_count(1); } while (0); __asm__ __volatile__("": : :"memory"); do { if (__builtin_constant_p((((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/spinlock_api_smp.h", .line = 153, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/spinlock_api_smp.h", .line = 153, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/spinlock_api_smp.h", .line = 153, }; ______r = !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/spinlock_api_smp.h", .line = 153, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) preempt_schedule(); } while (0); } while (0); } static inline __attribute__((always_inline)) void __raw_spin_unlock_irqrestore(raw_spinlock_t *lock, unsigned long flags) { lock_release(&lock->dep_map, 1, (unsigned long)__builtin_return_address(0)); do_raw_spin_unlock(lock); do { if (__builtin_constant_p(((({ ({ unsigned long __dummy; typeof(flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); arch_irqs_disabled_flags(flags); })))) ? !!((({ ({ unsigned long __dummy; typeof(flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); arch_irqs_disabled_flags(flags); }))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/spinlock_api_smp.h", .line = 161, }; ______r = !!((({ ({ unsigned long __dummy; typeof(flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); arch_irqs_disabled_flags(flags); }))); ______f.miss_hit[______r]++; ______r; })) { do { ({ unsigned long __dummy; typeof(flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); arch_local_irq_restore(flags); } while (0); trace_hardirqs_off(); } else { trace_hardirqs_on(); do { ({ unsigned long __dummy; typeof(flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); arch_local_irq_restore(flags); } while (0); } } while (0); do { do { __asm__ __volatile__("": : :"memory"); sub_preempt_count(1); } while (0); __asm__ __volatile__("": : :"memory"); do { if (__builtin_constant_p((((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/spinlock_api_smp.h", .line = 162, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/spinlock_api_smp.h", .line = 162, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/spinlock_api_smp.h", .line = 162, }; ______r = !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/spinlock_api_smp.h", .line = 162, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) preempt_schedule(); } while (0); } while (0); } static inline __attribute__((always_inline)) void __raw_spin_unlock_irq(raw_spinlock_t *lock) { lock_release(&lock->dep_map, 1, (unsigned long)__builtin_return_address(0)); do_raw_spin_unlock(lock); do { trace_hardirqs_on(); arch_local_irq_enable(); } while (0); do { do { __asm__ __volatile__("": : :"memory"); sub_preempt_count(1); } while (0); __asm__ __volatile__("": : :"memory"); do { if (__builtin_constant_p((((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/spinlock_api_smp.h", .line = 170, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/spinlock_api_smp.h", .line = 170, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/spinlock_api_smp.h", .line = 170, }; ______r = !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/spinlock_api_smp.h", .line = 170, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) preempt_schedule(); } while (0); } while (0); } static inline __attribute__((always_inline)) void __raw_spin_unlock_bh(raw_spinlock_t *lock) { lock_release(&lock->dep_map, 1, (unsigned long)__builtin_return_address(0)); do_raw_spin_unlock(lock); do { __asm__ __volatile__("": : :"memory"); sub_preempt_count(1); } while (0); local_bh_enable_ip((unsigned long)__builtin_return_address(0)); } static inline __attribute__((always_inline)) int __raw_spin_trylock_bh(raw_spinlock_t *lock) { local_bh_disable(); do { add_preempt_count(1); __asm__ __volatile__("": : :"memory"); } while (0); if (__builtin_constant_p(((do_raw_spin_trylock(lock)))) ? !!((do_raw_spin_trylock(lock))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/spinlock_api_smp.h", .line = 185, }; ______r = !!((do_raw_spin_trylock(lock))); ______f.miss_hit[______r]++; ______r; })) { lock_acquire(&lock->dep_map, 0, 1, 0, 2, ((void *)0), (unsigned long)__builtin_return_address(0)); return 1; } do { __asm__ __volatile__("": : :"memory"); sub_preempt_count(1); } while (0); local_bh_enable_ip((unsigned long)__builtin_return_address(0)); return 0; } void __attribute__((section(".spinlock.text"))) _raw_read_lock(rwlock_t *lock) ; void __attribute__((section(".spinlock.text"))) _raw_write_lock(rwlock_t *lock) ; void __attribute__((section(".spinlock.text"))) _raw_read_lock_bh(rwlock_t *lock) ; void __attribute__((section(".spinlock.text"))) _raw_write_lock_bh(rwlock_t *lock) ; void __attribute__((section(".spinlock.text"))) _raw_read_lock_irq(rwlock_t *lock) ; void __attribute__((section(".spinlock.text"))) _raw_write_lock_irq(rwlock_t *lock) ; unsigned long __attribute__((section(".spinlock.text"))) _raw_read_lock_irqsave(rwlock_t *lock) ; unsigned long __attribute__((section(".spinlock.text"))) _raw_write_lock_irqsave(rwlock_t *lock) ; int __attribute__((section(".spinlock.text"))) _raw_read_trylock(rwlock_t *lock); int __attribute__((section(".spinlock.text"))) _raw_write_trylock(rwlock_t *lock); void __attribute__((section(".spinlock.text"))) _raw_read_unlock(rwlock_t *lock) ; void __attribute__((section(".spinlock.text"))) _raw_write_unlock(rwlock_t *lock) ; void __attribute__((section(".spinlock.text"))) _raw_read_unlock_bh(rwlock_t *lock) ; void __attribute__((section(".spinlock.text"))) _raw_write_unlock_bh(rwlock_t *lock) ; void __attribute__((section(".spinlock.text"))) _raw_read_unlock_irq(rwlock_t *lock) ; void __attribute__((section(".spinlock.text"))) _raw_write_unlock_irq(rwlock_t *lock) ; void __attribute__((section(".spinlock.text"))) _raw_read_unlock_irqrestore(rwlock_t *lock, unsigned long flags) ; void __attribute__((section(".spinlock.text"))) _raw_write_unlock_irqrestore(rwlock_t *lock, unsigned long flags) ; static inline __attribute__((always_inline)) int __raw_read_trylock(rwlock_t *lock) { do { add_preempt_count(1); __asm__ __volatile__("": : :"memory"); } while (0); if (__builtin_constant_p(((do_raw_read_trylock(lock)))) ? !!((do_raw_read_trylock(lock))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/rwlock_api_smp.h", .line = 120, }; ______r = !!((do_raw_read_trylock(lock))); ______f.miss_hit[______r]++; ______r; })) { lock_acquire(&lock->dep_map, 0, 1, 2, 2, ((void *)0), (unsigned long)__builtin_return_address(0)); return 1; } do { do { __asm__ __volatile__("": : :"memory"); sub_preempt_count(1); } while (0); __asm__ __volatile__("": : :"memory"); do { if (__builtin_constant_p((((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/rwlock_api_smp.h", .line = 124, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/rwlock_api_smp.h", .line = 124, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/rwlock_api_smp.h", .line = 124, }; ______r = !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/rwlock_api_smp.h", .line = 124, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) preempt_schedule(); } while (0); } while (0); return 0; } static inline __attribute__((always_inline)) int __raw_write_trylock(rwlock_t *lock) { do { add_preempt_count(1); __asm__ __volatile__("": : :"memory"); } while (0); if (__builtin_constant_p(((do_raw_write_trylock(lock)))) ? !!((do_raw_write_trylock(lock))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/rwlock_api_smp.h", .line = 131, }; ______r = !!((do_raw_write_trylock(lock))); ______f.miss_hit[______r]++; ______r; })) { lock_acquire(&lock->dep_map, 0, 1, 0, 2, ((void *)0), (unsigned long)__builtin_return_address(0)); return 1; } do { do { __asm__ __volatile__("": : :"memory"); sub_preempt_count(1); } while (0); __asm__ __volatile__("": : :"memory"); do { if (__builtin_constant_p((((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/rwlock_api_smp.h", .line = 135, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/rwlock_api_smp.h", .line = 135, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/rwlock_api_smp.h", .line = 135, }; ______r = !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/rwlock_api_smp.h", .line = 135, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) preempt_schedule(); } while (0); } while (0); return 0; } static inline __attribute__((always_inline)) void __raw_read_lock(rwlock_t *lock) { do { add_preempt_count(1); __asm__ __volatile__("": : :"memory"); } while (0); lock_acquire(&lock->dep_map, 0, 0, 2, 2, ((void *)0), (unsigned long)__builtin_return_address(0)); do_raw_read_lock(lock); } static inline __attribute__((always_inline)) unsigned long __raw_read_lock_irqsave(rwlock_t *lock) { unsigned long flags; do { do { ({ unsigned long __dummy; typeof(flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); flags = arch_local_irq_save(); } while (0); trace_hardirqs_off(); } while (0); do { add_preempt_count(1); __asm__ __volatile__("": : :"memory"); } while (0); lock_acquire(&lock->dep_map, 0, 0, 2, 2, ((void *)0), (unsigned long)__builtin_return_address(0)); (do_raw_read_lock)((lock)) ; return flags; } static inline __attribute__((always_inline)) void __raw_read_lock_irq(rwlock_t *lock) { do { arch_local_irq_disable(); trace_hardirqs_off(); } while (0); do { add_preempt_count(1); __asm__ __volatile__("": : :"memory"); } while (0); lock_acquire(&lock->dep_map, 0, 0, 2, 2, ((void *)0), (unsigned long)__builtin_return_address(0)); do_raw_read_lock(lock); } static inline __attribute__((always_inline)) void __raw_read_lock_bh(rwlock_t *lock) { local_bh_disable(); do { add_preempt_count(1); __asm__ __volatile__("": : :"memory"); } while (0); lock_acquire(&lock->dep_map, 0, 0, 2, 2, ((void *)0), (unsigned long)__builtin_return_address(0)); do_raw_read_lock(lock); } static inline __attribute__((always_inline)) unsigned long __raw_write_lock_irqsave(rwlock_t *lock) { unsigned long flags; do { do { ({ unsigned long __dummy; typeof(flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); flags = arch_local_irq_save(); } while (0); trace_hardirqs_off(); } while (0); do { add_preempt_count(1); __asm__ __volatile__("": : :"memory"); } while (0); lock_acquire(&lock->dep_map, 0, 0, 0, 2, ((void *)0), (unsigned long)__builtin_return_address(0)); (do_raw_write_lock)((lock)) ; return flags; } static inline __attribute__((always_inline)) void __raw_write_lock_irq(rwlock_t *lock) { do { arch_local_irq_disable(); trace_hardirqs_off(); } while (0); do { add_preempt_count(1); __asm__ __volatile__("": : :"memory"); } while (0); lock_acquire(&lock->dep_map, 0, 0, 0, 2, ((void *)0), (unsigned long)__builtin_return_address(0)); do_raw_write_lock(lock); } static inline __attribute__((always_inline)) void __raw_write_lock_bh(rwlock_t *lock) { local_bh_disable(); do { add_preempt_count(1); __asm__ __volatile__("": : :"memory"); } while (0); lock_acquire(&lock->dep_map, 0, 0, 0, 2, ((void *)0), (unsigned long)__builtin_return_address(0)); do_raw_write_lock(lock); } static inline __attribute__((always_inline)) void __raw_write_lock(rwlock_t *lock) { do { add_preempt_count(1); __asm__ __volatile__("": : :"memory"); } while (0); lock_acquire(&lock->dep_map, 0, 0, 0, 2, ((void *)0), (unsigned long)__builtin_return_address(0)); do_raw_write_lock(lock); } static inline __attribute__((always_inline)) void __raw_write_unlock(rwlock_t *lock) { lock_release(&lock->dep_map, 1, (unsigned long)__builtin_return_address(0)); do_raw_write_unlock(lock); do { do { __asm__ __volatile__("": : :"memory"); sub_preempt_count(1); } while (0); __asm__ __volatile__("": : :"memory"); do { if (__builtin_constant_p((((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/rwlock_api_smp.h", .line = 222, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/rwlock_api_smp.h", .line = 222, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/rwlock_api_smp.h", .line = 222, }; ______r = !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/rwlock_api_smp.h", .line = 222, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) preempt_schedule(); } while (0); } while (0); } static inline __attribute__((always_inline)) void __raw_read_unlock(rwlock_t *lock) { lock_release(&lock->dep_map, 1, (unsigned long)__builtin_return_address(0)); do_raw_read_unlock(lock); do { do { __asm__ __volatile__("": : :"memory"); sub_preempt_count(1); } while (0); __asm__ __volatile__("": : :"memory"); do { if (__builtin_constant_p((((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/rwlock_api_smp.h", .line = 229, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/rwlock_api_smp.h", .line = 229, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/rwlock_api_smp.h", .line = 229, }; ______r = !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/rwlock_api_smp.h", .line = 229, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) preempt_schedule(); } while (0); } while (0); } static inline __attribute__((always_inline)) void __raw_read_unlock_irqrestore(rwlock_t *lock, unsigned long flags) { lock_release(&lock->dep_map, 1, (unsigned long)__builtin_return_address(0)); do_raw_read_unlock(lock); do { if (__builtin_constant_p(((({ ({ unsigned long __dummy; typeof(flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); arch_irqs_disabled_flags(flags); })))) ? !!((({ ({ unsigned long __dummy; typeof(flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); arch_irqs_disabled_flags(flags); }))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/rwlock_api_smp.h", .line = 237, }; ______r = !!((({ ({ unsigned long __dummy; typeof(flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); arch_irqs_disabled_flags(flags); }))); ______f.miss_hit[______r]++; ______r; })) { do { ({ unsigned long __dummy; typeof(flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); arch_local_irq_restore(flags); } while (0); trace_hardirqs_off(); } else { trace_hardirqs_on(); do { ({ unsigned long __dummy; typeof(flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); arch_local_irq_restore(flags); } while (0); } } while (0); do { do { __asm__ __volatile__("": : :"memory"); sub_preempt_count(1); } while (0); __asm__ __volatile__("": : :"memory"); do { if (__builtin_constant_p((((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/rwlock_api_smp.h", .line = 238, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/rwlock_api_smp.h", .line = 238, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/rwlock_api_smp.h", .line = 238, }; ______r = !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/rwlock_api_smp.h", .line = 238, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) preempt_schedule(); } while (0); } while (0); } static inline __attribute__((always_inline)) void __raw_read_unlock_irq(rwlock_t *lock) { lock_release(&lock->dep_map, 1, (unsigned long)__builtin_return_address(0)); do_raw_read_unlock(lock); do { trace_hardirqs_on(); arch_local_irq_enable(); } while (0); do { do { __asm__ __volatile__("": : :"memory"); sub_preempt_count(1); } while (0); __asm__ __volatile__("": : :"memory"); do { if (__builtin_constant_p((((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/rwlock_api_smp.h", .line = 246, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/rwlock_api_smp.h", .line = 246, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/rwlock_api_smp.h", .line = 246, }; ______r = !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/rwlock_api_smp.h", .line = 246, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) preempt_schedule(); } while (0); } while (0); } static inline __attribute__((always_inline)) void __raw_read_unlock_bh(rwlock_t *lock) { lock_release(&lock->dep_map, 1, (unsigned long)__builtin_return_address(0)); do_raw_read_unlock(lock); do { __asm__ __volatile__("": : :"memory"); sub_preempt_count(1); } while (0); local_bh_enable_ip((unsigned long)__builtin_return_address(0)); } static inline __attribute__((always_inline)) void __raw_write_unlock_irqrestore(rwlock_t *lock, unsigned long flags) { lock_release(&lock->dep_map, 1, (unsigned long)__builtin_return_address(0)); do_raw_write_unlock(lock); do { if (__builtin_constant_p(((({ ({ unsigned long __dummy; typeof(flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); arch_irqs_disabled_flags(flags); })))) ? !!((({ ({ unsigned long __dummy; typeof(flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); arch_irqs_disabled_flags(flags); }))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/rwlock_api_smp.h", .line = 262, }; ______r = !!((({ ({ unsigned long __dummy; typeof(flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); arch_irqs_disabled_flags(flags); }))); ______f.miss_hit[______r]++; ______r; })) { do { ({ unsigned long __dummy; typeof(flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); arch_local_irq_restore(flags); } while (0); trace_hardirqs_off(); } else { trace_hardirqs_on(); do { ({ unsigned long __dummy; typeof(flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); arch_local_irq_restore(flags); } while (0); } } while (0); do { do { __asm__ __volatile__("": : :"memory"); sub_preempt_count(1); } while (0); __asm__ __volatile__("": : :"memory"); do { if (__builtin_constant_p((((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/rwlock_api_smp.h", .line = 263, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/rwlock_api_smp.h", .line = 263, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/rwlock_api_smp.h", .line = 263, }; ______r = !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/rwlock_api_smp.h", .line = 263, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) preempt_schedule(); } while (0); } while (0); } static inline __attribute__((always_inline)) void __raw_write_unlock_irq(rwlock_t *lock) { lock_release(&lock->dep_map, 1, (unsigned long)__builtin_return_address(0)); do_raw_write_unlock(lock); do { trace_hardirqs_on(); arch_local_irq_enable(); } while (0); do { do { __asm__ __volatile__("": : :"memory"); sub_preempt_count(1); } while (0); __asm__ __volatile__("": : :"memory"); do { if (__builtin_constant_p((((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/rwlock_api_smp.h", .line = 271, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/rwlock_api_smp.h", .line = 271, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/rwlock_api_smp.h", .line = 271, }; ______r = !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/rwlock_api_smp.h", .line = 271, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) preempt_schedule(); } while (0); } while (0); } static inline __attribute__((always_inline)) void __raw_write_unlock_bh(rwlock_t *lock) { lock_release(&lock->dep_map, 1, (unsigned long)__builtin_return_address(0)); do_raw_write_unlock(lock); do { __asm__ __volatile__("": : :"memory"); sub_preempt_count(1); } while (0); local_bh_enable_ip((unsigned long)__builtin_return_address(0)); } static inline __attribute__((always_inline)) raw_spinlock_t *spinlock_check(spinlock_t *lock) { return &lock->rlock; } static inline __attribute__((always_inline)) void spin_lock(spinlock_t *lock) { _raw_spin_lock(&lock->rlock); } static inline __attribute__((always_inline)) void spin_lock_bh(spinlock_t *lock) { _raw_spin_lock_bh(&lock->rlock); } static inline __attribute__((always_inline)) int spin_trylock(spinlock_t *lock) { return (_raw_spin_trylock(&lock->rlock)); } static inline __attribute__((always_inline)) void spin_lock_irq(spinlock_t *lock) { _raw_spin_lock_irq(&lock->rlock); } static inline __attribute__((always_inline)) void spin_unlock(spinlock_t *lock) { _raw_spin_unlock(&lock->rlock); } static inline __attribute__((always_inline)) void spin_unlock_bh(spinlock_t *lock) { _raw_spin_unlock_bh(&lock->rlock); } static inline __attribute__((always_inline)) void spin_unlock_irq(spinlock_t *lock) { _raw_spin_unlock_irq(&lock->rlock); } static inline __attribute__((always_inline)) void spin_unlock_irqrestore(spinlock_t *lock, unsigned long flags) { do { ({ unsigned long __dummy; typeof(flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); _raw_spin_unlock_irqrestore(&lock->rlock, flags); } while (0); } static inline __attribute__((always_inline)) int spin_trylock_bh(spinlock_t *lock) { return (_raw_spin_trylock_bh(&lock->rlock)); } static inline __attribute__((always_inline)) int spin_trylock_irq(spinlock_t *lock) { return ({ do { arch_local_irq_disable(); trace_hardirqs_off(); } while (0); (_raw_spin_trylock(&lock->rlock)) ? 1 : ({ do { trace_hardirqs_on(); arch_local_irq_enable(); } while (0); 0; }); }); } static inline __attribute__((always_inline)) void spin_unlock_wait(spinlock_t *lock) { arch_spin_unlock_wait(&(&lock->rlock)->raw_lock); } static inline __attribute__((always_inline)) int spin_is_locked(spinlock_t *lock) { return arch_spin_is_locked(&(&lock->rlock)->raw_lock); } static inline __attribute__((always_inline)) int spin_is_contended(spinlock_t *lock) { return arch_spin_is_contended(&(&lock->rlock)->raw_lock); } static inline __attribute__((always_inline)) int spin_can_lock(spinlock_t *lock) { return (!arch_spin_is_locked(&(&lock->rlock)->raw_lock)); } static inline __attribute__((always_inline)) void assert_spin_locked(spinlock_t *lock) { do { if (__builtin_constant_p((((__builtin_constant_p(!arch_spin_is_locked(&(&lock->rlock)->raw_lock)) ? !!(!arch_spin_is_locked(&(&lock->rlock)->raw_lock)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/spinlock.h", .line = 380, }; ______r = __builtin_expect(!!(!arch_spin_is_locked(&(&lock->rlock)->raw_lock)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(!arch_spin_is_locked(&(&lock->rlock)->raw_lock)) ? !!(!arch_spin_is_locked(&(&lock->rlock)->raw_lock)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/spinlock.h", .line = 380, }; ______r = __builtin_expect(!!(!arch_spin_is_locked(&(&lock->rlock)->raw_lock)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/spinlock.h", .line = 380, }; ______r = !!(((__builtin_constant_p(!arch_spin_is_locked(&(&lock->rlock)->raw_lock)) ? !!(!arch_spin_is_locked(&(&lock->rlock)->raw_lock)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/spinlock.h", .line = 380, }; ______r = __builtin_expect(!!(!arch_spin_is_locked(&(&lock->rlock)->raw_lock)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) do { asm volatile("1:\tud2\n" ".pushsection __bug_table,\"a\"\n" "2:\t.long 1b, %c0\n" "\t.word %c1, 0\n" "\t.org 2b+%c2\n" ".popsection" : : "i" ("include/linux/spinlock.h"), "i" (380), "i" (sizeof(struct bug_entry))); __builtin_unreachable(); } while (0); } while(0); } extern int _atomic_dec_and_lock(atomic_t *atomic, spinlock_t *lock); typedef struct { unsigned sequence; spinlock_t lock; } seqlock_t; static inline __attribute__((always_inline)) void write_seqlock(seqlock_t *sl) { spin_lock(&sl->lock); ++sl->sequence; __asm__ __volatile__("": : :"memory"); } static inline __attribute__((always_inline)) void write_sequnlock(seqlock_t *sl) { __asm__ __volatile__("": : :"memory"); sl->sequence++; spin_unlock(&sl->lock); } static inline __attribute__((always_inline)) int write_tryseqlock(seqlock_t *sl) { int ret = spin_trylock(&sl->lock); if (__builtin_constant_p(((ret))) ? !!((ret)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/seqlock.h", .line = 76, }; ______r = !!((ret)); ______f.miss_hit[______r]++; ______r; })) { ++sl->sequence; __asm__ __volatile__("": : :"memory"); } return ret; } static inline __attribute__((always_inline)) __attribute__((always_inline)) unsigned read_seqbegin(const seqlock_t *sl) { unsigned ret; repeat: ret = (*(volatile typeof(sl->sequence) *)&(sl->sequence)); if (__builtin_constant_p((((__builtin_constant_p(ret & 1) ? !!(ret & 1) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/seqlock.h", .line = 90, }; ______r = __builtin_expect(!!(ret & 1), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(ret & 1) ? !!(ret & 1) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/seqlock.h", .line = 90, }; ______r = __builtin_expect(!!(ret & 1), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/seqlock.h", .line = 90, }; ______r = !!(((__builtin_constant_p(ret & 1) ? !!(ret & 1) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/seqlock.h", .line = 90, }; ______r = __builtin_expect(!!(ret & 1), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) { cpu_relax(); goto repeat; } __asm__ __volatile__("": : :"memory"); return ret; } static inline __attribute__((always_inline)) __attribute__((always_inline)) int read_seqretry(const seqlock_t *sl, unsigned start) { __asm__ __volatile__("": : :"memory"); return (__builtin_constant_p(sl->sequence != start) ? !!(sl->sequence != start) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/seqlock.h", .line = 108, }; ______r = __builtin_expect(!!(sl->sequence != start), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })); } typedef struct seqcount { unsigned sequence; } seqcount_t; static inline __attribute__((always_inline)) unsigned __read_seqcount_begin(const seqcount_t *s) { unsigned ret; repeat: ret = s->sequence; if (__builtin_constant_p((((__builtin_constant_p(ret & 1) ? !!(ret & 1) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/seqlock.h", .line = 145, }; ______r = __builtin_expect(!!(ret & 1), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(ret & 1) ? !!(ret & 1) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/seqlock.h", .line = 145, }; ______r = __builtin_expect(!!(ret & 1), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/seqlock.h", .line = 145, }; ______r = !!(((__builtin_constant_p(ret & 1) ? !!(ret & 1) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/seqlock.h", .line = 145, }; ______r = __builtin_expect(!!(ret & 1), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) { cpu_relax(); goto repeat; } return ret; } static inline __attribute__((always_inline)) unsigned read_seqcount_begin(const seqcount_t *s) { unsigned ret = __read_seqcount_begin(s); __asm__ __volatile__("": : :"memory"); return ret; } static inline __attribute__((always_inline)) int __read_seqcount_retry(const seqcount_t *s, unsigned start) { return (__builtin_constant_p(s->sequence != start) ? !!(s->sequence != start) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/seqlock.h", .line = 184, }; ______r = __builtin_expect(!!(s->sequence != start), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })); } static inline __attribute__((always_inline)) int read_seqcount_retry(const seqcount_t *s, unsigned start) { __asm__ __volatile__("": : :"memory"); return __read_seqcount_retry(s, start); } static inline __attribute__((always_inline)) void write_seqcount_begin(seqcount_t *s) { s->sequence++; __asm__ __volatile__("": : :"memory"); } static inline __attribute__((always_inline)) void write_seqcount_end(seqcount_t *s) { __asm__ __volatile__("": : :"memory"); s->sequence++; } static inline __attribute__((always_inline)) void write_seqcount_barrier(seqcount_t *s) { __asm__ __volatile__("": : :"memory"); s->sequence+=2; } struct timespec { __kernel_time_t tv_sec; long tv_nsec; }; struct timeval { __kernel_time_t tv_sec; __kernel_suseconds_t tv_usec; }; struct timezone { int tz_minuteswest; int tz_dsttime; }; extern struct timezone sys_tz; static inline __attribute__((always_inline)) int timespec_equal(const struct timespec *a, const struct timespec *b) { return (a->tv_sec == b->tv_sec) && (a->tv_nsec == b->tv_nsec); } static inline __attribute__((always_inline)) int timespec_compare(const struct timespec *lhs, const struct timespec *rhs) { if (__builtin_constant_p(((lhs->tv_sec < rhs->tv_sec))) ? !!((lhs->tv_sec < rhs->tv_sec)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/time.h", .line = 58, }; ______r = !!((lhs->tv_sec < rhs->tv_sec)); ______f.miss_hit[______r]++; ______r; })) return -1; if (__builtin_constant_p(((lhs->tv_sec > rhs->tv_sec))) ? !!((lhs->tv_sec > rhs->tv_sec)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/time.h", .line = 60, }; ______r = !!((lhs->tv_sec > rhs->tv_sec)); ______f.miss_hit[______r]++; ______r; })) return 1; return lhs->tv_nsec - rhs->tv_nsec; } static inline __attribute__((always_inline)) int timeval_compare(const struct timeval *lhs, const struct timeval *rhs) { if (__builtin_constant_p(((lhs->tv_sec < rhs->tv_sec))) ? !!((lhs->tv_sec < rhs->tv_sec)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/time.h", .line = 67, }; ______r = !!((lhs->tv_sec < rhs->tv_sec)); ______f.miss_hit[______r]++; ______r; })) return -1; if (__builtin_constant_p(((lhs->tv_sec > rhs->tv_sec))) ? !!((lhs->tv_sec > rhs->tv_sec)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/time.h", .line = 69, }; ______r = !!((lhs->tv_sec > rhs->tv_sec)); ______f.miss_hit[______r]++; ______r; })) return 1; return lhs->tv_usec - rhs->tv_usec; } extern unsigned long mktime(const unsigned int year, const unsigned int mon, const unsigned int day, const unsigned int hour, const unsigned int min, const unsigned int sec); extern void set_normalized_timespec(struct timespec *ts, time_t sec, s64 nsec); extern struct timespec timespec_add_safe(const struct timespec lhs, const struct timespec rhs); static inline __attribute__((always_inline)) struct timespec timespec_add(struct timespec lhs, struct timespec rhs) { struct timespec ts_delta; set_normalized_timespec(&ts_delta, lhs.tv_sec + rhs.tv_sec, lhs.tv_nsec + rhs.tv_nsec); return ts_delta; } static inline __attribute__((always_inline)) struct timespec timespec_sub(struct timespec lhs, struct timespec rhs) { struct timespec ts_delta; set_normalized_timespec(&ts_delta, lhs.tv_sec - rhs.tv_sec, lhs.tv_nsec - rhs.tv_nsec); return ts_delta; } extern void read_persistent_clock(struct timespec *ts); extern void read_boot_clock(struct timespec *ts); extern int update_persistent_clock(struct timespec now); extern int no_sync_cmos_clock __attribute__((__section__(".data..read_mostly"))); void timekeeping_init(void); extern int timekeeping_suspended; unsigned long get_seconds(void); struct timespec current_kernel_time(void); struct timespec __current_kernel_time(void); struct timespec get_monotonic_coarse(void); void get_xtime_and_monotonic_and_sleep_offset(struct timespec *xtim, struct timespec *wtom, struct timespec *sleep); void timekeeping_inject_sleeptime(struct timespec *delta); static inline __attribute__((always_inline)) u32 arch_gettimeoffset(void) { return 0; } extern void do_gettimeofday(struct timeval *tv); extern int do_settimeofday(const struct timespec *tv); extern int do_sys_settimeofday(const struct timespec *tv, const struct timezone *tz); extern long do_utimes(int dfd, const char *filename, struct timespec *times, int flags); struct itimerval; extern int do_setitimer(int which, struct itimerval *value, struct itimerval *ovalue); extern unsigned int alarm_setitimer(unsigned int seconds); extern int do_getitimer(int which, struct itimerval *value); extern void getnstimeofday(struct timespec *tv); extern void getrawmonotonic(struct timespec *ts); extern void getnstime_raw_and_real(struct timespec *ts_raw, struct timespec *ts_real); extern void getboottime(struct timespec *ts); extern void monotonic_to_bootbased(struct timespec *ts); extern void get_monotonic_boottime(struct timespec *ts); extern struct timespec timespec_trunc(struct timespec t, unsigned gran); extern int timekeeping_valid_for_hres(void); extern u64 timekeeping_max_deferment(void); extern void timekeeping_leap_insert(int leapsecond); extern int timekeeping_inject_offset(struct timespec *ts); struct tms; extern void do_sys_times(struct tms *); struct tm { int tm_sec; int tm_min; int tm_hour; int tm_mday; int tm_mon; long tm_year; int tm_wday; int tm_yday; }; void time_to_tm(time_t totalsecs, int offset, struct tm *result); static inline __attribute__((always_inline)) s64 timespec_to_ns(const struct timespec *ts) { return ((s64) ts->tv_sec * 1000000000L) + ts->tv_nsec; } static inline __attribute__((always_inline)) s64 timeval_to_ns(const struct timeval *tv) { return ((s64) tv->tv_sec * 1000000000L) + tv->tv_usec * 1000L; } extern struct timespec ns_to_timespec(const s64 nsec); extern struct timeval ns_to_timeval(const s64 nsec); static inline __attribute__((always_inline)) __attribute__((always_inline)) void timespec_add_ns(struct timespec *a, u64 ns) { a->tv_sec += __iter_div_u64_rem(a->tv_nsec + ns, 1000000000L, &ns); a->tv_nsec = ns; } struct itimerspec { struct timespec it_interval; struct timespec it_value; }; struct itimerval { struct timeval it_interval; struct timeval it_value; }; struct timex { unsigned int modes; long offset; long freq; long maxerror; long esterror; int status; long constant; long precision; long tolerance; struct timeval time; long tick; long ppsfreq; long jitter; int shift; long stabil; long jitcnt; long calcnt; long errcnt; long stbcnt; int tai; int :32; int :32; int :32; int :32; int :32; int :32; int :32; int :32; int :32; int :32; int :32; }; typedef unsigned long long cycles_t; extern unsigned int cpu_khz; extern unsigned int tsc_khz; extern void disable_TSC(void); static inline __attribute__((always_inline)) cycles_t get_cycles(void) { unsigned long long ret = 0; (ret = paravirt_read_tsc()); return ret; } static inline __attribute__((always_inline)) __attribute__((always_inline)) cycles_t vget_cycles(void) { return (cycles_t)__native_read_tsc(); } extern void tsc_init(void); extern void mark_tsc_unstable(char *reason); extern int unsynchronized_tsc(void); extern int check_tsc_unstable(void); extern unsigned long native_calibrate_tsc(void); extern void check_tsc_sync_source(int cpu); extern void check_tsc_sync_target(void); extern int notsc_setup(char *); extern void save_sched_clock_state(void); extern void restore_sched_clock_state(void); extern unsigned long tick_usec; extern unsigned long tick_nsec; extern int time_status; extern void ntp_init(void); extern void ntp_clear(void); static inline __attribute__((always_inline)) int ntp_synced(void) { return !(time_status & 0x0040); } extern u64 tick_length; extern void second_overflow(void); extern void update_ntp_one_tick(void); extern int do_adjtimex(struct timex *); extern void hardpps(const struct timespec *, const struct timespec *); int read_current_timer(unsigned long *timer_val); extern u64 __attribute__((section(".data"))) jiffies_64; extern unsigned long volatile __attribute__((section(".data"))) jiffies; u64 get_jiffies_64(void); extern unsigned long preset_lpj; extern unsigned int jiffies_to_msecs(const unsigned long j); extern unsigned int jiffies_to_usecs(const unsigned long j); extern unsigned long msecs_to_jiffies(const unsigned int m); extern unsigned long usecs_to_jiffies(const unsigned int u); extern unsigned long timespec_to_jiffies(const struct timespec *value); extern void jiffies_to_timespec(const unsigned long jiffies, struct timespec *value); extern unsigned long timeval_to_jiffies(const struct timeval *value); extern void jiffies_to_timeval(const unsigned long jiffies, struct timeval *value); extern clock_t jiffies_to_clock_t(long x); extern unsigned long clock_t_to_jiffies(unsigned long x); extern u64 jiffies_64_to_clock_t(u64 x); extern u64 nsec_to_clock_t(u64 x); extern u64 nsecs_to_jiffies64(u64 n); extern unsigned long nsecs_to_jiffies(u64 n); union ktime { s64 tv64; }; typedef union ktime ktime_t; static inline __attribute__((always_inline)) ktime_t ktime_set(const long secs, const unsigned long nsecs) { return (ktime_t) { .tv64 = (s64)secs * 1000000000L + (s64)nsecs }; } static inline __attribute__((always_inline)) ktime_t timespec_to_ktime(struct timespec ts) { return ktime_set(ts.tv_sec, ts.tv_nsec); } static inline __attribute__((always_inline)) ktime_t timeval_to_ktime(struct timeval tv) { return ktime_set(tv.tv_sec, tv.tv_usec * 1000L); } static inline __attribute__((always_inline)) int ktime_equal(const ktime_t cmp1, const ktime_t cmp2) { return cmp1.tv64 == cmp2.tv64; } static inline __attribute__((always_inline)) s64 ktime_to_us(const ktime_t kt) { struct timeval tv = ns_to_timeval((kt).tv64); return (s64) tv.tv_sec * 1000000L + tv.tv_usec; } static inline __attribute__((always_inline)) s64 ktime_to_ms(const ktime_t kt) { struct timeval tv = ns_to_timeval((kt).tv64); return (s64) tv.tv_sec * 1000L + tv.tv_usec / 1000L; } static inline __attribute__((always_inline)) s64 ktime_us_delta(const ktime_t later, const ktime_t earlier) { return ktime_to_us(({ (ktime_t){ .tv64 = (later).tv64 - (earlier).tv64 }; })); } static inline __attribute__((always_inline)) ktime_t ktime_add_us(const ktime_t kt, const u64 usec) { return ({ (ktime_t){ .tv64 = (kt).tv64 + (usec * 1000) }; }); } static inline __attribute__((always_inline)) ktime_t ktime_sub_us(const ktime_t kt, const u64 usec) { return ({ (ktime_t){ .tv64 = (kt).tv64 - (usec * 1000) }; }); } extern ktime_t ktime_add_safe(const ktime_t lhs, const ktime_t rhs); extern void ktime_get_ts(struct timespec *ts); static inline __attribute__((always_inline)) ktime_t ns_to_ktime(u64 ns) { static const ktime_t ktime_zero = { .tv64 = 0 }; return ({ (ktime_t){ .tv64 = (ktime_zero).tv64 + (ns) }; }); } enum debug_obj_state { ODEBUG_STATE_NONE, ODEBUG_STATE_INIT, ODEBUG_STATE_INACTIVE, ODEBUG_STATE_ACTIVE, ODEBUG_STATE_DESTROYED, ODEBUG_STATE_NOTAVAILABLE, ODEBUG_STATE_MAX, }; struct debug_obj_descr; struct debug_obj { struct hlist_node node; enum debug_obj_state state; unsigned int astate; void *object; struct debug_obj_descr *descr; }; struct debug_obj_descr { const char *name; void *(*debug_hint) (void *addr); int (*fixup_init) (void *addr, enum debug_obj_state state); int (*fixup_activate) (void *addr, enum debug_obj_state state); int (*fixup_destroy) (void *addr, enum debug_obj_state state); int (*fixup_free) (void *addr, enum debug_obj_state state); }; static inline __attribute__((always_inline)) void debug_object_init (void *addr, struct debug_obj_descr *descr) { } static inline __attribute__((always_inline)) void debug_object_init_on_stack(void *addr, struct debug_obj_descr *descr) { } static inline __attribute__((always_inline)) void debug_object_activate (void *addr, struct debug_obj_descr *descr) { } static inline __attribute__((always_inline)) void debug_object_deactivate(void *addr, struct debug_obj_descr *descr) { } static inline __attribute__((always_inline)) void debug_object_destroy (void *addr, struct debug_obj_descr *descr) { } static inline __attribute__((always_inline)) void debug_object_free (void *addr, struct debug_obj_descr *descr) { } static inline __attribute__((always_inline)) void debug_objects_early_init(void) { } static inline __attribute__((always_inline)) void debug_objects_mem_init(void) { } static inline __attribute__((always_inline)) void debug_check_no_obj_freed(const void *address, unsigned long size) { } struct tvec_base; struct timer_list { struct list_head entry; unsigned long expires; struct tvec_base *base; void (*function)(unsigned long); unsigned long data; int slack; int start_pid; void *start_site; char start_comm[16]; struct lockdep_map lockdep_map; }; extern struct tvec_base boot_tvec_bases; void init_timer_key(struct timer_list *timer, const char *name, struct lock_class_key *key); void init_timer_deferrable_key(struct timer_list *timer, const char *name, struct lock_class_key *key); static inline __attribute__((always_inline)) void destroy_timer_on_stack(struct timer_list *timer) { } static inline __attribute__((always_inline)) void init_timer_on_stack_key(struct timer_list *timer, const char *name, struct lock_class_key *key) { init_timer_key(timer, name, key); } static inline __attribute__((always_inline)) void setup_timer_key(struct timer_list * timer, const char *name, struct lock_class_key *key, void (*function)(unsigned long), unsigned long data) { timer->function = function; timer->data = data; init_timer_key(timer, name, key); } static inline __attribute__((always_inline)) void setup_timer_on_stack_key(struct timer_list *timer, const char *name, struct lock_class_key *key, void (*function)(unsigned long), unsigned long data) { timer->function = function; timer->data = data; init_timer_on_stack_key(timer, name, key); } extern void setup_deferrable_timer_on_stack_key(struct timer_list *timer, const char *name, struct lock_class_key *key, void (*function)(unsigned long), unsigned long data); static inline __attribute__((always_inline)) int timer_pending(const struct timer_list * timer) { return timer->entry.next != ((void *)0); } extern void add_timer_on(struct timer_list *timer, int cpu); extern int del_timer(struct timer_list * timer); extern int mod_timer(struct timer_list *timer, unsigned long expires); extern int mod_timer_pending(struct timer_list *timer, unsigned long expires); extern int mod_timer_pinned(struct timer_list *timer, unsigned long expires); extern void set_timer_slack(struct timer_list *time, int slack_hz); extern unsigned long get_next_timer_interrupt(unsigned long now); extern int timer_stats_active; extern void init_timer_stats(void); extern void timer_stats_update_stats(void *timer, pid_t pid, void *startf, void *timerf, char *comm, unsigned int timer_flag); extern void __timer_stats_timer_set_start_info(struct timer_list *timer, void *addr); static inline __attribute__((always_inline)) void timer_stats_timer_set_start_info(struct timer_list *timer) { if (__builtin_constant_p((((__builtin_constant_p(!timer_stats_active) ? !!(!timer_stats_active) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/timer.h", .line = 252, }; ______r = __builtin_expect(!!(!timer_stats_active), 1); ftrace_likely_update(&______f, ______r, 1); ______r; }))))) ? !!(((__builtin_constant_p(!timer_stats_active) ? !!(!timer_stats_active) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/timer.h", .line = 252, }; ______r = __builtin_expect(!!(!timer_stats_active), 1); ftrace_likely_update(&______f, ______r, 1); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/timer.h", .line = 252, }; ______r = !!(((__builtin_constant_p(!timer_stats_active) ? !!(!timer_stats_active) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/timer.h", .line = 252, }; ______r = __builtin_expect(!!(!timer_stats_active), 1); ftrace_likely_update(&______f, ______r, 1); ______r; })))); ______f.miss_hit[______r]++; ______r; })) return; __timer_stats_timer_set_start_info(timer, __builtin_return_address(0)); } static inline __attribute__((always_inline)) void timer_stats_timer_clear_start_info(struct timer_list *timer) { timer->start_site = ((void *)0); } extern void add_timer(struct timer_list *timer); extern int try_to_del_timer_sync(struct timer_list *timer); extern int del_timer_sync(struct timer_list *timer); extern void init_timers(void); extern void run_local_timers(void); struct hrtimer; extern enum hrtimer_restart it_real_fn(struct hrtimer *); unsigned long __round_jiffies(unsigned long j, int cpu); unsigned long __round_jiffies_relative(unsigned long j, int cpu); unsigned long round_jiffies(unsigned long j); unsigned long round_jiffies_relative(unsigned long j); unsigned long __round_jiffies_up(unsigned long j, int cpu); unsigned long __round_jiffies_up_relative(unsigned long j, int cpu); unsigned long round_jiffies_up(unsigned long j); unsigned long round_jiffies_up_relative(unsigned long j); struct workqueue_struct; struct work_struct; typedef void (*work_func_t)(struct work_struct *work); enum { WORK_STRUCT_PENDING_BIT = 0, WORK_STRUCT_DELAYED_BIT = 1, WORK_STRUCT_CWQ_BIT = 2, WORK_STRUCT_LINKED_BIT = 3, WORK_STRUCT_COLOR_SHIFT = 4, WORK_STRUCT_COLOR_BITS = 4, WORK_STRUCT_PENDING = 1 << WORK_STRUCT_PENDING_BIT, WORK_STRUCT_DELAYED = 1 << WORK_STRUCT_DELAYED_BIT, WORK_STRUCT_CWQ = 1 << WORK_STRUCT_CWQ_BIT, WORK_STRUCT_LINKED = 1 << WORK_STRUCT_LINKED_BIT, WORK_STRUCT_STATIC = 0, WORK_NR_COLORS = (1 << WORK_STRUCT_COLOR_BITS) - 1, WORK_NO_COLOR = WORK_NR_COLORS, WORK_CPU_UNBOUND = 8, WORK_CPU_NONE = 8 + 1, WORK_CPU_LAST = WORK_CPU_NONE, WORK_STRUCT_FLAG_BITS = WORK_STRUCT_COLOR_SHIFT + WORK_STRUCT_COLOR_BITS, WORK_STRUCT_FLAG_MASK = (1UL << WORK_STRUCT_FLAG_BITS) - 1, WORK_STRUCT_WQ_DATA_MASK = ~WORK_STRUCT_FLAG_MASK, WORK_STRUCT_NO_CPU = WORK_CPU_NONE << WORK_STRUCT_FLAG_BITS, WORK_BUSY_PENDING = 1 << 0, WORK_BUSY_RUNNING = 1 << 1, }; struct work_struct { atomic_long_t data; struct list_head entry; work_func_t func; struct lockdep_map lockdep_map; }; struct delayed_work { struct work_struct work; struct timer_list timer; }; static inline __attribute__((always_inline)) struct delayed_work *to_delayed_work(struct work_struct *work) { return ({ const typeof( ((struct delayed_work *)0)->work ) *__mptr = (work); (struct delayed_work *)( (char *)__mptr - __builtin_offsetof(struct delayed_work,work) );}); } struct execute_work { struct work_struct work; }; static inline __attribute__((always_inline)) void __init_work(struct work_struct *work, int onstack) { } static inline __attribute__((always_inline)) void destroy_work_on_stack(struct work_struct *work) { } static inline __attribute__((always_inline)) unsigned int work_static(struct work_struct *work) { return 0; } enum { WQ_NON_REENTRANT = 1 << 0, WQ_UNBOUND = 1 << 1, WQ_FREEZABLE = 1 << 2, WQ_MEM_RECLAIM = 1 << 3, WQ_HIGHPRI = 1 << 4, WQ_CPU_INTENSIVE = 1 << 5, WQ_DYING = 1 << 6, WQ_RESCUER = 1 << 7, WQ_MAX_ACTIVE = 512, WQ_MAX_UNBOUND_PER_CPU = 4, WQ_DFL_ACTIVE = WQ_MAX_ACTIVE / 2, }; extern struct workqueue_struct *system_wq; extern struct workqueue_struct *system_long_wq; extern struct workqueue_struct *system_nrt_wq; extern struct workqueue_struct *system_unbound_wq; extern struct workqueue_struct *system_freezable_wq; extern struct workqueue_struct * __alloc_workqueue_key(const char *name, unsigned int flags, int max_active, struct lock_class_key *key, const char *lock_name); static inline __attribute__((always_inline)) struct workqueue_struct * alloc_ordered_workqueue(const char *name, unsigned int flags) { return ({ static struct lock_class_key __key; const char *__lock_name; if (__builtin_constant_p(((__builtin_constant_p(name)))) ? !!((__builtin_constant_p(name))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/workqueue.h", .line = 337, }; ______r = !!((__builtin_constant_p(name))); ______f.miss_hit[______r]++; ______r; })) __lock_name = (name); else __lock_name = "name"; __alloc_workqueue_key((name), (WQ_UNBOUND | flags), (1), &__key, __lock_name); }); } extern void destroy_workqueue(struct workqueue_struct *wq); extern int queue_work(struct workqueue_struct *wq, struct work_struct *work); extern int queue_work_on(int cpu, struct workqueue_struct *wq, struct work_struct *work); extern int queue_delayed_work(struct workqueue_struct *wq, struct delayed_work *work, unsigned long delay); extern int queue_delayed_work_on(int cpu, struct workqueue_struct *wq, struct delayed_work *work, unsigned long delay); extern void flush_workqueue(struct workqueue_struct *wq); extern void flush_scheduled_work(void); extern int schedule_work(struct work_struct *work); extern int schedule_work_on(int cpu, struct work_struct *work); extern int schedule_delayed_work(struct delayed_work *work, unsigned long delay); extern int schedule_delayed_work_on(int cpu, struct delayed_work *work, unsigned long delay); extern int schedule_on_each_cpu(work_func_t func); extern int keventd_up(void); int execute_in_process_context(work_func_t fn, struct execute_work *); extern bool flush_work(struct work_struct *work); extern bool flush_work_sync(struct work_struct *work); extern bool cancel_work_sync(struct work_struct *work); extern bool flush_delayed_work(struct delayed_work *dwork); extern bool flush_delayed_work_sync(struct delayed_work *work); extern bool cancel_delayed_work_sync(struct delayed_work *dwork); extern void workqueue_set_max_active(struct workqueue_struct *wq, int max_active); extern bool workqueue_congested(unsigned int cpu, struct workqueue_struct *wq); extern unsigned int work_cpu(struct work_struct *work); extern unsigned int work_busy(struct work_struct *work); static inline __attribute__((always_inline)) bool cancel_delayed_work(struct delayed_work *work) { bool ret; ret = del_timer_sync(&work->timer); if (__builtin_constant_p(((ret))) ? !!((ret)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/workqueue.h", .line = 395, }; ______r = !!((ret)); ______f.miss_hit[______r]++; ______r; })) clear_bit(WORK_STRUCT_PENDING_BIT, ((unsigned long *)(&(&work->work)->data))); return ret; } static inline __attribute__((always_inline)) bool __cancel_delayed_work(struct delayed_work *work) { bool ret; ret = del_timer(&work->timer); if (__builtin_constant_p(((ret))) ? !!((ret)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/workqueue.h", .line = 410, }; ______r = !!((ret)); ______f.miss_hit[______r]++; ______r; })) clear_bit(WORK_STRUCT_PENDING_BIT, ((unsigned long *)(&(&work->work)->data))); return ret; } static inline __attribute__((always_inline)) __attribute__((deprecated)) void cancel_rearming_delayed_workqueue(struct workqueue_struct *wq, struct delayed_work *work) { cancel_delayed_work_sync(work); } static inline __attribute__((always_inline)) __attribute__((deprecated)) void cancel_rearming_delayed_work(struct delayed_work *work) { cancel_delayed_work_sync(work); } long work_on_cpu(unsigned int cpu, long (*fn)(void *), void *arg); extern void freeze_workqueues_begin(void); extern bool freeze_workqueues_busy(void); extern void thaw_workqueues(void); typedef struct __wait_queue wait_queue_t; typedef int (*wait_queue_func_t)(wait_queue_t *wait, unsigned mode, int flags, void *key); int default_wake_function(wait_queue_t *wait, unsigned mode, int flags, void *key); struct __wait_queue { unsigned int flags; void *private; wait_queue_func_t func; struct list_head task_list; }; struct wait_bit_key { void *flags; int bit_nr; }; struct wait_bit_queue { struct wait_bit_key key; wait_queue_t wait; }; struct __wait_queue_head { spinlock_t lock; struct list_head task_list; }; typedef struct __wait_queue_head wait_queue_head_t; struct task_struct; extern void __init_waitqueue_head(wait_queue_head_t *q, struct lock_class_key *); static inline __attribute__((always_inline)) void init_waitqueue_entry(wait_queue_t *q, struct task_struct *p) { q->flags = 0; q->private = p; q->func = default_wake_function; } static inline __attribute__((always_inline)) void init_waitqueue_func_entry(wait_queue_t *q, wait_queue_func_t func) { q->flags = 0; q->private = ((void *)0); q->func = func; } static inline __attribute__((always_inline)) int waitqueue_active(wait_queue_head_t *q) { return !list_empty(&q->task_list); } extern void add_wait_queue(wait_queue_head_t *q, wait_queue_t *wait); extern void add_wait_queue_exclusive(wait_queue_head_t *q, wait_queue_t *wait); extern void remove_wait_queue(wait_queue_head_t *q, wait_queue_t *wait); static inline __attribute__((always_inline)) void __add_wait_queue(wait_queue_head_t *head, wait_queue_t *new) { list_add(&new->task_list, &head->task_list); } static inline __attribute__((always_inline)) void __add_wait_queue_exclusive(wait_queue_head_t *q, wait_queue_t *wait) { wait->flags |= 0x01; __add_wait_queue(q, wait); } static inline __attribute__((always_inline)) void __add_wait_queue_tail(wait_queue_head_t *head, wait_queue_t *new) { list_add_tail(&new->task_list, &head->task_list); } static inline __attribute__((always_inline)) void __add_wait_queue_tail_exclusive(wait_queue_head_t *q, wait_queue_t *wait) { wait->flags |= 0x01; __add_wait_queue_tail(q, wait); } static inline __attribute__((always_inline)) void __remove_wait_queue(wait_queue_head_t *head, wait_queue_t *old) { list_del(&old->task_list); } void __wake_up(wait_queue_head_t *q, unsigned int mode, int nr, void *key); void __wake_up_locked_key(wait_queue_head_t *q, unsigned int mode, void *key); void __wake_up_sync_key(wait_queue_head_t *q, unsigned int mode, int nr, void *key); void __wake_up_locked(wait_queue_head_t *q, unsigned int mode); void __wake_up_sync(wait_queue_head_t *q, unsigned int mode, int nr); void __wake_up_bit(wait_queue_head_t *, void *, int); int __wait_on_bit(wait_queue_head_t *, struct wait_bit_queue *, int (*)(void *), unsigned); int __wait_on_bit_lock(wait_queue_head_t *, struct wait_bit_queue *, int (*)(void *), unsigned); void wake_up_bit(void *, int); int out_of_line_wait_on_bit(void *, int, int (*)(void *), unsigned); int out_of_line_wait_on_bit_lock(void *, int, int (*)(void *), unsigned); wait_queue_head_t *bit_waitqueue(void *, int); extern void sleep_on(wait_queue_head_t *q); extern long sleep_on_timeout(wait_queue_head_t *q, signed long timeout); extern void interruptible_sleep_on(wait_queue_head_t *q); extern long interruptible_sleep_on_timeout(wait_queue_head_t *q, signed long timeout); void prepare_to_wait(wait_queue_head_t *q, wait_queue_t *wait, int state); void prepare_to_wait_exclusive(wait_queue_head_t *q, wait_queue_t *wait, int state); void finish_wait(wait_queue_head_t *q, wait_queue_t *wait); void abort_exclusive_wait(wait_queue_head_t *q, wait_queue_t *wait, unsigned int mode, void *key); int autoremove_wake_function(wait_queue_t *wait, unsigned mode, int sync, void *key); int wake_bit_function(wait_queue_t *wait, unsigned mode, int sync, void *key); static inline __attribute__((always_inline)) int wait_on_bit(void *word, int bit, int (*action)(void *), unsigned mode) { if (__builtin_constant_p(((!(__builtin_constant_p((bit)) ? constant_test_bit((bit), (word)) : variable_test_bit((bit), (word)))))) ? !!((!(__builtin_constant_p((bit)) ? constant_test_bit((bit), (word)) : variable_test_bit((bit), (word))))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/wait.h", .line = 637, }; ______r = !!((!(__builtin_constant_p((bit)) ? constant_test_bit((bit), (word)) : variable_test_bit((bit), (word))))); ______f.miss_hit[______r]++; ______r; })) return 0; return out_of_line_wait_on_bit(word, bit, action, mode); } static inline __attribute__((always_inline)) int wait_on_bit_lock(void *word, int bit, int (*action)(void *), unsigned mode) { if (__builtin_constant_p(((!test_and_set_bit(bit, word)))) ? !!((!test_and_set_bit(bit, word))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/wait.h", .line = 661, }; ______r = !!((!test_and_set_bit(bit, word))); ______f.miss_hit[______r]++; ______r; })) return 0; return out_of_line_wait_on_bit_lock(word, bit, action, mode); } struct completion { unsigned int done; wait_queue_head_t wait; }; static inline __attribute__((always_inline)) void init_completion(struct completion *x) { x->done = 0; do { static struct lock_class_key __key; __init_waitqueue_head((&x->wait), &__key); } while (0); } extern void wait_for_completion(struct completion *); extern int wait_for_completion_interruptible(struct completion *x); extern int wait_for_completion_killable(struct completion *x); extern unsigned long wait_for_completion_timeout(struct completion *x, unsigned long timeout); extern long wait_for_completion_interruptible_timeout( struct completion *x, unsigned long timeout); extern long wait_for_completion_killable_timeout( struct completion *x, unsigned long timeout); extern bool try_wait_for_completion(struct completion *x); extern bool completion_done(struct completion *x); extern void complete(struct completion *); extern void complete_all(struct completion *); extern void (*pm_idle)(void); extern void (*pm_power_off)(void); extern void (*pm_power_off_prepare)(void); struct device; extern const char power_group_name[]; typedef struct pm_message { int event; } pm_message_t; struct dev_pm_ops { int (*prepare)(struct device *dev); void (*complete)(struct device *dev); int (*suspend)(struct device *dev); int (*resume)(struct device *dev); int (*freeze)(struct device *dev); int (*thaw)(struct device *dev); int (*poweroff)(struct device *dev); int (*restore)(struct device *dev); int (*suspend_noirq)(struct device *dev); int (*resume_noirq)(struct device *dev); int (*freeze_noirq)(struct device *dev); int (*thaw_noirq)(struct device *dev); int (*poweroff_noirq)(struct device *dev); int (*restore_noirq)(struct device *dev); int (*runtime_suspend)(struct device *dev); int (*runtime_resume)(struct device *dev); int (*runtime_idle)(struct device *dev); }; extern struct dev_pm_ops generic_subsys_pm_ops; enum rpm_status { RPM_ACTIVE = 0, RPM_RESUMING, RPM_SUSPENDED, RPM_SUSPENDING, }; enum rpm_request { RPM_REQ_NONE = 0, RPM_REQ_IDLE, RPM_REQ_SUSPEND, RPM_REQ_AUTOSUSPEND, RPM_REQ_RESUME, }; struct wakeup_source; struct dev_pm_info { pm_message_t power_state; unsigned int can_wakeup:1; unsigned int async_suspend:1; bool is_prepared:1; bool is_suspended:1; spinlock_t lock; struct list_head entry; struct completion completion; struct wakeup_source *wakeup; struct timer_list suspend_timer; unsigned long timer_expires; struct work_struct work; wait_queue_head_t wait_queue; atomic_t usage_count; atomic_t child_count; unsigned int disable_depth:3; unsigned int ignore_children:1; unsigned int idle_notification:1; unsigned int request_pending:1; unsigned int deferred_resume:1; unsigned int run_wake:1; unsigned int runtime_auto:1; unsigned int no_callbacks:1; unsigned int irq_safe:1; unsigned int use_autosuspend:1; unsigned int timer_autosuspends:1; enum rpm_request request; enum rpm_status runtime_status; int runtime_error; int autosuspend_delay; unsigned long last_busy; unsigned long active_jiffies; unsigned long suspended_jiffies; unsigned long accounting_timestamp; void *subsys_data; }; extern void update_pm_runtime_accounting(struct device *dev); struct dev_power_domain { struct dev_pm_ops ops; }; extern void device_pm_lock(void); extern void dpm_resume_noirq(pm_message_t state); extern void dpm_resume_end(pm_message_t state); extern void dpm_resume(pm_message_t state); extern void dpm_complete(pm_message_t state); extern void device_pm_unlock(void); extern int dpm_suspend_noirq(pm_message_t state); extern int dpm_suspend_start(pm_message_t state); extern int dpm_suspend(pm_message_t state); extern int dpm_prepare(pm_message_t state); extern void __suspend_report_result(const char *function, void *fn, int ret); extern int device_pm_wait_for_dev(struct device *sub, struct device *dev); extern int pm_generic_prepare(struct device *dev); extern int pm_generic_suspend(struct device *dev); extern int pm_generic_resume(struct device *dev); extern int pm_generic_freeze(struct device *dev); extern int pm_generic_thaw(struct device *dev); extern int pm_generic_restore(struct device *dev); extern int pm_generic_poweroff(struct device *dev); extern void pm_generic_complete(struct device *dev); enum dpm_order { DPM_ORDER_NONE, DPM_ORDER_DEV_AFTER_PARENT, DPM_ORDER_PARENT_BEFORE_DEV, DPM_ORDER_DEV_LAST, }; typedef struct { unsigned long bits[((((1 << 0)) + (8 * sizeof(long)) - 1) / (8 * sizeof(long)))]; } nodemask_t; extern nodemask_t _unused_nodemask_arg_; static inline __attribute__((always_inline)) void __node_set(int node, volatile nodemask_t *dstp) { set_bit(node, dstp->bits); } static inline __attribute__((always_inline)) void __node_clear(int node, volatile nodemask_t *dstp) { clear_bit(node, dstp->bits); } static inline __attribute__((always_inline)) void __nodes_setall(nodemask_t *dstp, int nbits) { bitmap_fill(dstp->bits, nbits); } static inline __attribute__((always_inline)) void __nodes_clear(nodemask_t *dstp, int nbits) { bitmap_zero(dstp->bits, nbits); } static inline __attribute__((always_inline)) int __node_test_and_set(int node, nodemask_t *addr) { return test_and_set_bit(node, addr->bits); } static inline __attribute__((always_inline)) void __nodes_and(nodemask_t *dstp, const nodemask_t *src1p, const nodemask_t *src2p, int nbits) { bitmap_and(dstp->bits, src1p->bits, src2p->bits, nbits); } static inline __attribute__((always_inline)) void __nodes_or(nodemask_t *dstp, const nodemask_t *src1p, const nodemask_t *src2p, int nbits) { bitmap_or(dstp->bits, src1p->bits, src2p->bits, nbits); } static inline __attribute__((always_inline)) void __nodes_xor(nodemask_t *dstp, const nodemask_t *src1p, const nodemask_t *src2p, int nbits) { bitmap_xor(dstp->bits, src1p->bits, src2p->bits, nbits); } static inline __attribute__((always_inline)) void __nodes_andnot(nodemask_t *dstp, const nodemask_t *src1p, const nodemask_t *src2p, int nbits) { bitmap_andnot(dstp->bits, src1p->bits, src2p->bits, nbits); } static inline __attribute__((always_inline)) void __nodes_complement(nodemask_t *dstp, const nodemask_t *srcp, int nbits) { bitmap_complement(dstp->bits, srcp->bits, nbits); } static inline __attribute__((always_inline)) int __nodes_equal(const nodemask_t *src1p, const nodemask_t *src2p, int nbits) { return bitmap_equal(src1p->bits, src2p->bits, nbits); } static inline __attribute__((always_inline)) int __nodes_intersects(const nodemask_t *src1p, const nodemask_t *src2p, int nbits) { return bitmap_intersects(src1p->bits, src2p->bits, nbits); } static inline __attribute__((always_inline)) int __nodes_subset(const nodemask_t *src1p, const nodemask_t *src2p, int nbits) { return bitmap_subset(src1p->bits, src2p->bits, nbits); } static inline __attribute__((always_inline)) int __nodes_empty(const nodemask_t *srcp, int nbits) { return bitmap_empty(srcp->bits, nbits); } static inline __attribute__((always_inline)) int __nodes_full(const nodemask_t *srcp, int nbits) { return bitmap_full(srcp->bits, nbits); } static inline __attribute__((always_inline)) int __nodes_weight(const nodemask_t *srcp, int nbits) { return bitmap_weight(srcp->bits, nbits); } static inline __attribute__((always_inline)) void __nodes_shift_right(nodemask_t *dstp, const nodemask_t *srcp, int n, int nbits) { bitmap_shift_right(dstp->bits, srcp->bits, n, nbits); } static inline __attribute__((always_inline)) void __nodes_shift_left(nodemask_t *dstp, const nodemask_t *srcp, int n, int nbits) { bitmap_shift_left(dstp->bits, srcp->bits, n, nbits); } static inline __attribute__((always_inline)) int __first_node(const nodemask_t *srcp) { return ({ int __min1 = ((1 << 0)); int __min2 = (find_first_bit(srcp->bits, (1 << 0))); __min1 < __min2 ? __min1: __min2; }); } static inline __attribute__((always_inline)) int __next_node(int n, const nodemask_t *srcp) { return ({ int __min1 = ((1 << 0)); int __min2 = (find_next_bit(srcp->bits, (1 << 0), n+1)); __min1 < __min2 ? __min1: __min2; }); } static inline __attribute__((always_inline)) void init_nodemask_of_node(nodemask_t *mask, int node) { __nodes_clear(&(*mask), (1 << 0)); __node_set((node), &(*mask)); } static inline __attribute__((always_inline)) int __first_unset_node(const nodemask_t *maskp) { return ({ int __min1 = ((1 << 0)); int __min2 = (find_first_zero_bit(maskp->bits, (1 << 0))); __min1 < __min2 ? __min1: __min2; }) ; } static inline __attribute__((always_inline)) int __nodemask_scnprintf(char *buf, int len, const nodemask_t *srcp, int nbits) { return bitmap_scnprintf(buf, len, srcp->bits, nbits); } static inline __attribute__((always_inline)) int __nodemask_parse_user(const char *buf, int len, nodemask_t *dstp, int nbits) { return bitmap_parse_user(buf, len, dstp->bits, nbits); } static inline __attribute__((always_inline)) int __nodelist_scnprintf(char *buf, int len, const nodemask_t *srcp, int nbits) { return bitmap_scnlistprintf(buf, len, srcp->bits, nbits); } static inline __attribute__((always_inline)) int __nodelist_parse(const char *buf, nodemask_t *dstp, int nbits) { return bitmap_parselist(buf, dstp->bits, nbits); } static inline __attribute__((always_inline)) int __node_remap(int oldbit, const nodemask_t *oldp, const nodemask_t *newp, int nbits) { return bitmap_bitremap(oldbit, oldp->bits, newp->bits, nbits); } static inline __attribute__((always_inline)) void __nodes_remap(nodemask_t *dstp, const nodemask_t *srcp, const nodemask_t *oldp, const nodemask_t *newp, int nbits) { bitmap_remap(dstp->bits, srcp->bits, oldp->bits, newp->bits, nbits); } static inline __attribute__((always_inline)) void __nodes_onto(nodemask_t *dstp, const nodemask_t *origp, const nodemask_t *relmapp, int nbits) { bitmap_onto(dstp->bits, origp->bits, relmapp->bits, nbits); } static inline __attribute__((always_inline)) void __nodes_fold(nodemask_t *dstp, const nodemask_t *origp, int sz, int nbits) { bitmap_fold(dstp->bits, origp->bits, sz, nbits); } enum node_states { N_POSSIBLE, N_ONLINE, N_NORMAL_MEMORY, N_HIGH_MEMORY, N_CPU, NR_NODE_STATES }; extern nodemask_t node_states[NR_NODE_STATES]; static inline __attribute__((always_inline)) int node_state(int node, enum node_states state) { return node == 0; } static inline __attribute__((always_inline)) void node_set_state(int node, enum node_states state) { } static inline __attribute__((always_inline)) void node_clear_state(int node, enum node_states state) { } static inline __attribute__((always_inline)) int num_node_state(enum node_states state) { return 1; } struct nodemask_scratch { nodemask_t mask1; nodemask_t mask2; }; static inline __attribute__((always_inline)) int numa_node_id(void) { return 0; } static inline __attribute__((always_inline)) int early_cpu_to_node(int cpu) { return 0; } static inline __attribute__((always_inline)) void setup_node_to_cpumask_map(void) { } extern const struct cpumask *cpu_coregroup_mask(int cpu); static inline __attribute__((always_inline)) void arch_fix_phys_package_id(int num, u32 slot) { } struct pci_bus; void x86_pci_root_bus_res_quirks(struct pci_bus *b); static inline __attribute__((always_inline)) int get_mp_bus_to_node(int busnum) { return 0; } static inline __attribute__((always_inline)) void set_mp_bus_to_node(int busnum, int node) { } static inline __attribute__((always_inline)) void set_apicid_to_node(int apicid, s16 node) { } static inline __attribute__((always_inline)) int numa_cpu_node(int cpu) { return (-1); } extern void set_highmem_pages_init(void); static inline __attribute__((always_inline)) void numa_set_node(int cpu, int node) { } static inline __attribute__((always_inline)) void numa_clear_node(int cpu) { } static inline __attribute__((always_inline)) void init_cpu_to_node(void) { } static inline __attribute__((always_inline)) void numa_add_cpu(int cpu) { } static inline __attribute__((always_inline)) void numa_remove_cpu(int cpu) { } struct mutex { atomic_t count; spinlock_t wait_lock; struct list_head wait_list; struct task_struct *owner; const char *name; void *magic; struct lockdep_map dep_map; }; struct mutex_waiter { struct list_head list; struct task_struct *task; void *magic; }; extern void mutex_destroy(struct mutex *lock); extern void __mutex_init(struct mutex *lock, const char *name, struct lock_class_key *key); static inline __attribute__((always_inline)) int mutex_is_locked(struct mutex *lock) { return atomic_read(&lock->count) != 1; } extern void mutex_lock_nested(struct mutex *lock, unsigned int subclass); extern void _mutex_lock_nest_lock(struct mutex *lock, struct lockdep_map *nest_lock); extern int __attribute__((warn_unused_result)) mutex_lock_interruptible_nested(struct mutex *lock, unsigned int subclass); extern int __attribute__((warn_unused_result)) mutex_lock_killable_nested(struct mutex *lock, unsigned int subclass); extern int mutex_trylock(struct mutex *lock); extern void mutex_unlock(struct mutex *lock); extern int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock); typedef struct { void *ldt; int size; struct mutex lock; void *vdso; } mm_context_t; void leave_mm(int cpu); enum xen_domain_type { XEN_NATIVE, XEN_PV_DOMAIN, XEN_HVM_DOMAIN, }; static inline __attribute__((always_inline)) unsigned char readb(const volatile void *addr) { unsigned char ret; asm volatile("mov" "b" " %1,%0":"=q" (ret) :"m" (*(volatile unsigned char *)addr) :"memory"); return ret; } static inline __attribute__((always_inline)) unsigned short readw(const volatile void *addr) { unsigned short ret; asm volatile("mov" "w" " %1,%0":"=r" (ret) :"m" (*(volatile unsigned short *)addr) :"memory"); return ret; } static inline __attribute__((always_inline)) unsigned int readl(const volatile void *addr) { unsigned int ret; asm volatile("mov" "l" " %1,%0":"=r" (ret) :"m" (*(volatile unsigned int *)addr) :"memory"); return ret; } static inline __attribute__((always_inline)) unsigned char __readb(const volatile void *addr) { unsigned char ret; asm volatile("mov" "b" " %1,%0":"=q" (ret) :"m" (*(volatile unsigned char *)addr) ); return ret; } static inline __attribute__((always_inline)) unsigned short __readw(const volatile void *addr) { unsigned short ret; asm volatile("mov" "w" " %1,%0":"=r" (ret) :"m" (*(volatile unsigned short *)addr) ); return ret; } static inline __attribute__((always_inline)) unsigned int __readl(const volatile void *addr) { unsigned int ret; asm volatile("mov" "l" " %1,%0":"=r" (ret) :"m" (*(volatile unsigned int *)addr) ); return ret; } static inline __attribute__((always_inline)) void writeb(unsigned char val, volatile void *addr) { asm volatile("mov" "b" " %0,%1": :"q" (val), "m" (*(volatile unsigned char *)addr) :"memory"); } static inline __attribute__((always_inline)) void writew(unsigned short val, volatile void *addr) { asm volatile("mov" "w" " %0,%1": :"r" (val), "m" (*(volatile unsigned short *)addr) :"memory"); } static inline __attribute__((always_inline)) void writel(unsigned int val, volatile void *addr) { asm volatile("mov" "l" " %0,%1": :"r" (val), "m" (*(volatile unsigned int *)addr) :"memory"); } static inline __attribute__((always_inline)) void __writeb(unsigned char val, volatile void *addr) { asm volatile("mov" "b" " %0,%1": :"q" (val), "m" (*(volatile unsigned char *)addr) ); } static inline __attribute__((always_inline)) void __writew(unsigned short val, volatile void *addr) { asm volatile("mov" "w" " %0,%1": :"r" (val), "m" (*(volatile unsigned short *)addr) ); } static inline __attribute__((always_inline)) void __writel(unsigned int val, volatile void *addr) { asm volatile("mov" "l" " %0,%1": :"r" (val), "m" (*(volatile unsigned int *)addr) ); } static inline __attribute__((always_inline)) phys_addr_t virt_to_phys(volatile void *address) { return (((unsigned long)(address)) - ((unsigned long)(0xC0000000UL))); } static inline __attribute__((always_inline)) void *phys_to_virt(phys_addr_t address) { return ((void *)((unsigned long)(address)+((unsigned long)(0xC0000000UL)))); } static inline __attribute__((always_inline)) unsigned int isa_virt_to_bus(volatile void *address) { return (unsigned int)virt_to_phys(address); } extern void *ioremap_nocache(resource_size_t offset, unsigned long size); extern void *ioremap_cache(resource_size_t offset, unsigned long size); extern void *ioremap_prot(resource_size_t offset, unsigned long size, unsigned long prot_val); static inline __attribute__((always_inline)) void *ioremap(resource_size_t offset, unsigned long size) { return ioremap_nocache(offset, size); } extern void iounmap(volatile void *addr); extern void set_iounmap_nonlazy(void); extern unsigned int ioread8(void *); extern unsigned int ioread16(void *); extern unsigned int ioread16be(void *); extern unsigned int ioread32(void *); extern unsigned int ioread32be(void *); extern void iowrite8(u8, void *); extern void iowrite16(u16, void *); extern void iowrite16be(u16, void *); extern void iowrite32(u32, void *); extern void iowrite32be(u32, void *); extern void ioread8_rep(void *port, void *buf, unsigned long count); extern void ioread16_rep(void *port, void *buf, unsigned long count); extern void ioread32_rep(void *port, void *buf, unsigned long count); extern void iowrite8_rep(void *port, const void *buf, unsigned long count); extern void iowrite16_rep(void *port, const void *buf, unsigned long count); extern void iowrite32_rep(void *port, const void *buf, unsigned long count); extern void *ioport_map(unsigned long port, unsigned int nr); extern void ioport_unmap(void *); struct pci_dev; extern void *pci_iomap(struct pci_dev *dev, int bar, unsigned long max); extern void pci_iounmap(struct pci_dev *dev, void *); struct vm_area_struct; struct vm_struct { struct vm_struct *next; void *addr; unsigned long size; unsigned long flags; struct page **pages; unsigned int nr_pages; phys_addr_t phys_addr; void *caller; }; extern void vm_unmap_ram(const void *mem, unsigned int count); extern void *vm_map_ram(struct page **pages, unsigned int count, int node, pgprot_t prot); extern void vm_unmap_aliases(void); extern void __attribute__ ((__section__(".init.text"))) __attribute__((__cold__)) __attribute__((no_instrument_function)) vmalloc_init(void); extern void *vmalloc(unsigned long size); extern void *vzalloc(unsigned long size); extern void *vmalloc_user(unsigned long size); extern void *vmalloc_node(unsigned long size, int node); extern void *vzalloc_node(unsigned long size, int node); extern void *vmalloc_exec(unsigned long size); extern void *vmalloc_32(unsigned long size); extern void *vmalloc_32_user(unsigned long size); extern void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot); extern void *__vmalloc_node_range(unsigned long size, unsigned long align, unsigned long start, unsigned long end, gfp_t gfp_mask, pgprot_t prot, int node, void *caller); extern void vfree(const void *addr); extern void *vmap(struct page **pages, unsigned int count, unsigned long flags, pgprot_t prot); extern void vunmap(const void *addr); extern int remap_vmalloc_range(struct vm_area_struct *vma, void *addr, unsigned long pgoff); void vmalloc_sync_all(void); static inline __attribute__((always_inline)) size_t get_vm_area_size(const struct vm_struct *area) { return area->size - ((1UL) << 12); } extern struct vm_struct *get_vm_area(unsigned long size, unsigned long flags); extern struct vm_struct *get_vm_area_caller(unsigned long size, unsigned long flags, void *caller); extern struct vm_struct *__get_vm_area(unsigned long size, unsigned long flags, unsigned long start, unsigned long end); extern struct vm_struct *__get_vm_area_caller(unsigned long size, unsigned long flags, unsigned long start, unsigned long end, void *caller); extern struct vm_struct *remove_vm_area(const void *addr); extern int map_vm_area(struct vm_struct *area, pgprot_t prot, struct page ***pages); extern int map_kernel_range_noflush(unsigned long start, unsigned long size, pgprot_t prot, struct page **pages); extern void unmap_kernel_range_noflush(unsigned long addr, unsigned long size); extern void unmap_kernel_range(unsigned long addr, unsigned long size); extern struct vm_struct *alloc_vm_area(size_t size); extern void free_vm_area(struct vm_struct *area); extern long vread(char *buf, char *addr, unsigned long count); extern long vwrite(char *buf, char *addr, unsigned long count); extern rwlock_t vmlist_lock; extern struct vm_struct *vmlist; extern __attribute__ ((__section__(".init.text"))) __attribute__((__cold__)) __attribute__((no_instrument_function)) void vm_area_register_early(struct vm_struct *vm, size_t align); struct vm_struct **pcpu_get_vm_areas(const unsigned long *offsets, const size_t *sizes, int nr_vms, size_t align); void pcpu_free_vm_areas(struct vm_struct **vms, int nr_vms); static inline __attribute__((always_inline)) void memset_io(volatile void *addr, unsigned char val, size_t count) { __builtin_memset((void *)addr, val, count); } static inline __attribute__((always_inline)) void memcpy_fromio(void *dst, const volatile void *src, size_t count) { __builtin_memcpy(dst, (const void *)src, count); } static inline __attribute__((always_inline)) void memcpy_toio(volatile void *dst, const void *src, size_t count) { __builtin_memcpy((void *)dst, src, count); } static inline __attribute__((always_inline)) void flush_write_buffers(void) { } extern void native_io_delay(void); extern int io_delay_type; extern void io_delay_init(void); static inline __attribute__((always_inline)) void outb(unsigned char value, int port) { asm volatile("out" "b" " %" "b" "0, %w1" : : "a"(value), "Nd"(port)); } static inline __attribute__((always_inline)) unsigned char inb(int port) { unsigned char value; asm volatile("in" "b" " %w1, %" "b" "0" : "=a"(value) : "Nd"(port)); return value; } static inline __attribute__((always_inline)) void outb_p(unsigned char value, int port) { outb(value, port); slow_down_io(); } static inline __attribute__((always_inline)) unsigned char inb_p(int port) { unsigned char value = inb(port); slow_down_io(); return value; } static inline __attribute__((always_inline)) void outsb(int port, const void *addr, unsigned long count) { asm volatile("rep; outs" "b" : "+S"(addr), "+c"(count) : "d"(port)); } static inline __attribute__((always_inline)) void insb(int port, void *addr, unsigned long count) { asm volatile("rep; ins" "b" : "+D"(addr), "+c"(count) : "d"(port)); } static inline __attribute__((always_inline)) void outw(unsigned short value, int port) { asm volatile("out" "w" " %" "w" "0, %w1" : : "a"(value), "Nd"(port)); } static inline __attribute__((always_inline)) unsigned short inw(int port) { unsigned short value; asm volatile("in" "w" " %w1, %" "w" "0" : "=a"(value) : "Nd"(port)); return value; } static inline __attribute__((always_inline)) void outw_p(unsigned short value, int port) { outw(value, port); slow_down_io(); } static inline __attribute__((always_inline)) unsigned short inw_p(int port) { unsigned short value = inw(port); slow_down_io(); return value; } static inline __attribute__((always_inline)) void outsw(int port, const void *addr, unsigned long count) { asm volatile("rep; outs" "w" : "+S"(addr), "+c"(count) : "d"(port)); } static inline __attribute__((always_inline)) void insw(int port, void *addr, unsigned long count) { asm volatile("rep; ins" "w" : "+D"(addr), "+c"(count) : "d"(port)); } static inline __attribute__((always_inline)) void outl(unsigned int value, int port) { asm volatile("out" "l" " %" "" "0, %w1" : : "a"(value), "Nd"(port)); } static inline __attribute__((always_inline)) unsigned int inl(int port) { unsigned int value; asm volatile("in" "l" " %w1, %" "" "0" : "=a"(value) : "Nd"(port)); return value; } static inline __attribute__((always_inline)) void outl_p(unsigned int value, int port) { outl(value, port); slow_down_io(); } static inline __attribute__((always_inline)) unsigned int inl_p(int port) { unsigned int value = inl(port); slow_down_io(); return value; } static inline __attribute__((always_inline)) void outsl(int port, const void *addr, unsigned long count) { asm volatile("rep; outs" "l" : "+S"(addr), "+c"(count) : "d"(port)); } static inline __attribute__((always_inline)) void insl(int port, void *addr, unsigned long count) { asm volatile("rep; ins" "l" : "+D"(addr), "+c"(count) : "d"(port)); } extern void *xlate_dev_mem_ptr(unsigned long phys); extern void unxlate_dev_mem_ptr(unsigned long phys, void *addr); extern int ioremap_change_attr(unsigned long vaddr, unsigned long size, unsigned long prot_val); extern void *ioremap_wc(resource_size_t offset, unsigned long size); extern void early_ioremap_init(void); extern void early_ioremap_reset(void); extern void *early_ioremap(resource_size_t phys_addr, unsigned long size); extern void *early_memremap(resource_size_t phys_addr, unsigned long size); extern void early_iounmap(void *addr, unsigned long size); extern void fixup_early_ioremap(void); extern bool is_early_ioremap_ptep(pte_t *ptep); extern const unsigned char x86_trampoline_start []; extern const unsigned char x86_trampoline_end []; extern unsigned char *x86_trampoline_base; extern unsigned long init_rsp; extern unsigned long initial_code; extern unsigned long initial_gs; extern void __attribute__ ((__section__(".init.text"))) __attribute__((__cold__)) __attribute__((no_instrument_function)) setup_trampolines(void); extern const unsigned char trampoline_data[]; extern const unsigned char trampoline_status[]; static inline __attribute__((always_inline)) unsigned long trampoline_address(void) { return virt_to_phys(((void *)(x86_trampoline_base + ((const unsigned char *)(trampoline_data) - x86_trampoline_start)))); } int __acpi_acquire_global_lock(unsigned int *lock); int __acpi_release_global_lock(unsigned int *lock); extern int acpi_lapic; extern int acpi_ioapic; extern int acpi_noirq; extern int acpi_strict; extern int acpi_disabled; extern int acpi_pci_disabled; extern int acpi_skip_timer_override; extern int acpi_use_timer_override; extern int acpi_fix_pin2_polarity; extern u8 acpi_sci_flags; extern int acpi_sci_override_gsi; void acpi_pic_sci_set_trigger(unsigned int, u16); extern int (*__acpi_register_gsi)(struct device *dev, u32 gsi, int trigger, int polarity); static inline __attribute__((always_inline)) void disable_acpi(void) { acpi_disabled = 1; acpi_pci_disabled = 1; acpi_noirq = 1; } extern int acpi_gsi_to_irq(u32 gsi, unsigned int *irq); static inline __attribute__((always_inline)) void acpi_noirq_set(void) { acpi_noirq = 1; } static inline __attribute__((always_inline)) void acpi_disable_pci(void) { acpi_pci_disabled = 1; acpi_noirq_set(); } extern int acpi_suspend_lowlevel(void); extern const unsigned char acpi_wakeup_code[]; extern void acpi_reserve_wakeup_memory(void); static inline __attribute__((always_inline)) unsigned int acpi_processor_cstate_check(unsigned int max_cstate) { if (__builtin_constant_p(((boot_cpu_data.x86 == 0x0F && boot_cpu_data.x86_vendor == 2 && boot_cpu_data.x86_model <= 0x05 && boot_cpu_data.x86_mask < 0x0A))) ? !!((boot_cpu_data.x86 == 0x0F && boot_cpu_data.x86_vendor == 2 && boot_cpu_data.x86_model <= 0x05 && boot_cpu_data.x86_mask < 0x0A)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/acpi.h" , .line = 140 , }; ______r = !!((boot_cpu_data.x86 == 0x0F && boot_cpu_data.x86_vendor == 2 && boot_cpu_data.x86_model <= 0x05 && boot_cpu_data.x86_mask < 0x0A)); ______f.miss_hit[______r]++; ______r; })) return 1; else if (__builtin_constant_p(((amd_e400_c1e_detected))) ? !!((amd_e400_c1e_detected)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/acpi.h", .line = 142, }; ______r = !!((amd_e400_c1e_detected)); ______f.miss_hit[______r]++; ______r; })) return 1; else return max_cstate; } static inline __attribute__((always_inline)) bool arch_has_acpi_pdc(void) { struct cpuinfo_x86 *c = &(*({ do { const void *__vpp_verify = (typeof((&(cpu_info))))((void *)0); (void)__vpp_verify; } while (0); ({ unsigned long __ptr; __asm__ ("" : "=r"(__ptr) : "0"((typeof(*(&(cpu_info))) *)(&(cpu_info)))); (typeof((typeof(*(&(cpu_info))) *)(&(cpu_info)))) (__ptr + (((__per_cpu_offset[0])))); }); })); return (c->x86_vendor == 0 || c->x86_vendor == 5); } static inline __attribute__((always_inline)) void arch_acpi_set_pdc_bits(u32 *buf) { struct cpuinfo_x86 *c = &(*({ do { const void *__vpp_verify = (typeof((&(cpu_info))))((void *)0); (void)__vpp_verify; } while (0); ({ unsigned long __ptr; __asm__ ("" : "=r"(__ptr) : "0"((typeof(*(&(cpu_info))) *)(&(cpu_info)))); (typeof((typeof(*(&(cpu_info))) *)(&(cpu_info)))) (__ptr + (((__per_cpu_offset[0])))); }); })); buf[2] |= ((0x0010) | (0x0008) | (0x0002) | (0x0100) | (0x0200)); if (__builtin_constant_p((((__builtin_constant_p((4*32+ 7)) && ( ((((4*32+ 7))>>5)==0 && (1UL<<(((4*32+ 7))&31) & ((1<<((0*32+ 0) & 31))|0|0|(1<<((0*32+ 6) & 31))| (1<<((0*32+ 8) & 31))|0|0|(1<<((0*32+15) & 31))| 0|0))) || ((((4*32+ 7))>>5)==1 && (1UL<<(((4*32+ 7))&31) & (0|0))) || ((((4*32+ 7))>>5)==2 && (1UL<<(((4*32+ 7))&31) & 0)) || ((((4*32+ 7))>>5)==3 && (1UL<<(((4*32+ 7))&31) & (0))) || ((((4*32+ 7))>>5)==4 && (1UL<<(((4*32+ 7))&31) & 0)) || ((((4*32+ 7))>>5)==5 && (1UL<<(((4*32+ 7))&31) & 0)) || ((((4*32+ 7))>>5)==6 && (1UL<<(((4*32+ 7))&31) & 0)) || ((((4*32+ 7))>>5)==7 && (1UL<<(((4*32+ 7))&31) & 0)) || ((((4*32+ 7))>>5)==8 && (1UL<<(((4*32+ 7))&31) & 0)) || ((((4*32+ 7))>>5)==9 && (1UL<<(((4*32+ 7))&31) & 0)) ) ? 1 : (__builtin_constant_p(((4*32+ 7))) ? constant_test_bit(((4*32+ 7)), ((unsigned long *)((c)->x86_capability))) : variable_test_bit(((4*32+ 7)), ((unsigned long *)((c)->x86_capability)))))))) ? !!(((__builtin_constant_p((4*32+ 7)) && ( ((((4*32+ 7))>>5)==0 && (1UL<<(((4*32+ 7))&31) & ((1<<((0*32+ 0) & 31))|0|0|(1<<((0*32+ 6) & 31))| (1<<((0*32+ 8) & 31))|0|0|(1<<((0*32+15) & 31))| 0|0))) || ((((4*32+ 7))>>5)==1 && (1UL<<(((4*32+ 7))&31) & (0|0))) || ((((4*32+ 7))>>5)==2 && (1UL<<(((4*32+ 7))&31) & 0)) || ((((4*32+ 7))>>5)==3 && (1UL<<(((4*32+ 7))&31) & (0))) || ((((4*32+ 7))>>5)==4 && (1UL<<(((4*32+ 7))&31) & 0)) || ((((4*32+ 7))>>5)==5 && (1UL<<(((4*32+ 7))&31) & 0)) || ((((4*32+ 7))>>5)==6 && (1UL<<(((4*32+ 7))&31) & 0)) || ((((4*32+ 7))>>5)==7 && (1UL<<(((4*32+ 7))&31) & 0)) || ((((4*32+ 7))>>5)==8 && (1UL<<(((4*32+ 7))&31) & 0)) || ((((4*32+ 7))>>5)==9 && (1UL<<(((4*32+ 7))&31) & 0)) ) ? 1 : (__builtin_constant_p(((4*32+ 7))) ? constant_test_bit(((4*32+ 7)), ((unsigned long *)((c)->x86_capability))) : variable_test_bit(((4*32+ 7)), ((unsigned long *)((c)->x86_capability))))))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/acpi.h", .line = 161, }; ______r = !!(((__builtin_constant_p((4*32+ 7)) && ( ((((4*32+ 7))>>5)==0 && (1UL<<(((4*32+ 7))&31) & ((1<<((0*32+ 0) & 31))|0|0|(1<<((0*32+ 6) & 31))| (1<<((0*32+ 8) & 31))|0|0|(1<<((0*32+15) & 31))| 0|0))) || ((((4*32+ 7))>>5)==1 && (1UL<<(((4*32+ 7))&31) & (0|0))) || ((((4*32+ 7))>>5)==2 && (1UL<<(((4*32+ 7))&31) & 0)) || ((((4*32+ 7))>>5)==3 && (1UL<<(((4*32+ 7))&31) & (0))) || ((((4*32+ 7))>>5)==4 && (1UL<<(((4*32+ 7))&31) & 0)) || ((((4*32+ 7))>>5)==5 && (1UL<<(((4*32+ 7))&31) & 0)) || ((((4*32+ 7))>>5)==6 && (1UL<<(((4*32+ 7))&31) & 0)) || ((((4*32+ 7))>>5)==7 && (1UL<<(((4*32+ 7))&31) & 0)) || ((((4*32+ 7))>>5)==8 && (1UL<<(((4*32+ 7))&31) & 0)) || ((((4*32+ 7))>>5)==9 && (1UL<<(((4*32+ 7))&31) & 0)) ) ? 1 : (__builtin_constant_p(((4*32+ 7))) ? constant_test_bit(((4*32+ 7)), ((unsigned long *)((c)->x86_capability))) : variable_test_bit(((4*32+ 7)), ((unsigned long *)((c)->x86_capability))))))); ______f.miss_hit[______r]++; ______r; })) buf[2] |= ((0x0008) | (0x0002) | (0x0020) | (0x0800) | (0x0001)); if (__builtin_constant_p((((__builtin_constant_p((0*32+22)) && ( ((((0*32+22))>>5)==0 && (1UL<<(((0*32+22))&31) & ((1<<((0*32+ 0) & 31))|0|0|(1<<((0*32+ 6) & 31))| (1<<((0*32+ 8) & 31))|0|0|(1<<((0*32+15) & 31))| 0|0))) || ((((0*32+22))>>5)==1 && (1UL<<(((0*32+22))&31) & (0|0))) || ((((0*32+22))>>5)==2 && (1UL<<(((0*32+22))&31) & 0)) || ((((0*32+22))>>5)==3 && (1UL<<(((0*32+22))&31) & (0))) || ((((0*32+22))>>5)==4 && (1UL<<(((0*32+22))&31) & 0)) || ((((0*32+22))>>5)==5 && (1UL<<(((0*32+22))&31) & 0)) || ((((0*32+22))>>5)==6 && (1UL<<(((0*32+22))&31) & 0)) || ((((0*32+22))>>5)==7 && (1UL<<(((0*32+22))&31) & 0)) || ((((0*32+22))>>5)==8 && (1UL<<(((0*32+22))&31) & 0)) || ((((0*32+22))>>5)==9 && (1UL<<(((0*32+22))&31) & 0)) ) ? 1 : (__builtin_constant_p(((0*32+22))) ? constant_test_bit(((0*32+22)), ((unsigned long *)((c)->x86_capability))) : variable_test_bit(((0*32+22)), ((unsigned long *)((c)->x86_capability)))))))) ? !!(((__builtin_constant_p((0*32+22)) && ( ((((0*32+22))>>5)==0 && (1UL<<(((0*32+22))&31) & ((1<<((0*32+ 0) & 31))|0|0|(1<<((0*32+ 6) & 31))| (1<<((0*32+ 8) & 31))|0|0|(1<<((0*32+15) & 31))| 0|0))) || ((((0*32+22))>>5)==1 && (1UL<<(((0*32+22))&31) & (0|0))) || ((((0*32+22))>>5)==2 && (1UL<<(((0*32+22))&31) & 0)) || ((((0*32+22))>>5)==3 && (1UL<<(((0*32+22))&31) & (0))) || ((((0*32+22))>>5)==4 && (1UL<<(((0*32+22))&31) & 0)) || ((((0*32+22))>>5)==5 && (1UL<<(((0*32+22))&31) & 0)) || ((((0*32+22))>>5)==6 && (1UL<<(((0*32+22))&31) & 0)) || ((((0*32+22))>>5)==7 && (1UL<<(((0*32+22))&31) & 0)) || ((((0*32+22))>>5)==8 && (1UL<<(((0*32+22))&31) & 0)) || ((((0*32+22))>>5)==9 && (1UL<<(((0*32+22))&31) & 0)) ) ? 1 : (__builtin_constant_p(((0*32+22))) ? constant_test_bit(((0*32+22)), ((unsigned long *)((c)->x86_capability))) : variable_test_bit(((0*32+22)), ((unsigned long *)((c)->x86_capability))))))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/acpi.h", .line = 164, }; ______r = !!(((__builtin_constant_p((0*32+22)) && ( ((((0*32+22))>>5)==0 && (1UL<<(((0*32+22))&31) & ((1<<((0*32+ 0) & 31))|0|0|(1<<((0*32+ 6) & 31))| (1<<((0*32+ 8) & 31))|0|0|(1<<((0*32+15) & 31))| 0|0))) || ((((0*32+22))>>5)==1 && (1UL<<(((0*32+22))&31) & (0|0))) || ((((0*32+22))>>5)==2 && (1UL<<(((0*32+22))&31) & 0)) || ((((0*32+22))>>5)==3 && (1UL<<(((0*32+22))&31) & (0))) || ((((0*32+22))>>5)==4 && (1UL<<(((0*32+22))&31) & 0)) || ((((0*32+22))>>5)==5 && (1UL<<(((0*32+22))&31) & 0)) || ((((0*32+22))>>5)==6 && (1UL<<(((0*32+22))&31) & 0)) || ((((0*32+22))>>5)==7 && (1UL<<(((0*32+22))&31) & 0)) || ((((0*32+22))>>5)==8 && (1UL<<(((0*32+22))&31) & 0)) || ((((0*32+22))>>5)==9 && (1UL<<(((0*32+22))&31) & 0)) ) ? 1 : (__builtin_constant_p(((0*32+22))) ? constant_test_bit(((0*32+22)), ((unsigned long *)((c)->x86_capability))) : variable_test_bit(((0*32+22)), ((unsigned long *)((c)->x86_capability))))))); ______f.miss_hit[______r]++; ______r; })) buf[2] |= (0x0004); if (__builtin_constant_p(((!(__builtin_constant_p((4*32+ 3)) && ( ((((4*32+ 3))>>5)==0 && (1UL<<(((4*32+ 3))&31) & ((1<<((0*32+ 0) & 31))|0|0|(1<<((0*32+ 6) & 31))| (1<<((0*32+ 8) & 31))|0|0|(1<<((0*32+15) & 31))| 0|0))) || ((((4*32+ 3))>>5)==1 && (1UL<<(((4*32+ 3))&31) & (0|0))) || ((((4*32+ 3))>>5)==2 && (1UL<<(((4*32+ 3))&31) & 0)) || ((((4*32+ 3))>>5)==3 && (1UL<<(((4*32+ 3))&31) & (0))) || ((((4*32+ 3))>>5)==4 && (1UL<<(((4*32+ 3))&31) & 0)) || ((((4*32+ 3))>>5)==5 && (1UL<<(((4*32+ 3))&31) & 0)) || ((((4*32+ 3))>>5)==6 && (1UL<<(((4*32+ 3))&31) & 0)) || ((((4*32+ 3))>>5)==7 && (1UL<<(((4*32+ 3))&31) & 0)) || ((((4*32+ 3))>>5)==8 && (1UL<<(((4*32+ 3))&31) & 0)) || ((((4*32+ 3))>>5)==9 && (1UL<<(((4*32+ 3))&31) & 0)) ) ? 1 : (__builtin_constant_p(((4*32+ 3))) ? constant_test_bit(((4*32+ 3)), ((unsigned long *)((c)->x86_capability))) : variable_test_bit(((4*32+ 3)), ((unsigned long *)((c)->x86_capability)))))))) ? !!((!(__builtin_constant_p((4*32+ 3)) && ( ((((4*32+ 3))>>5)==0 && (1UL<<(((4*32+ 3))&31) & ((1<<((0*32+ 0) & 31))|0|0|(1<<((0*32+ 6) & 31))| (1<<((0*32+ 8) & 31))|0|0|(1<<((0*32+15) & 31))| 0|0))) || ((((4*32+ 3))>>5)==1 && (1UL<<(((4*32+ 3))&31) & (0|0))) || ((((4*32+ 3))>>5)==2 && (1UL<<(((4*32+ 3))&31) & 0)) || ((((4*32+ 3))>>5)==3 && (1UL<<(((4*32+ 3))&31) & (0))) || ((((4*32+ 3))>>5)==4 && (1UL<<(((4*32+ 3))&31) & 0)) || ((((4*32+ 3))>>5)==5 && (1UL<<(((4*32+ 3))&31) & 0)) || ((((4*32+ 3))>>5)==6 && (1UL<<(((4*32+ 3))&31) & 0)) || ((((4*32+ 3))>>5)==7 && (1UL<<(((4*32+ 3))&31) & 0)) || ((((4*32+ 3))>>5)==8 && (1UL<<(((4*32+ 3))&31) & 0)) || ((((4*32+ 3))>>5)==9 && (1UL<<(((4*32+ 3))&31) & 0)) ) ? 1 : (__builtin_constant_p(((4*32+ 3))) ? constant_test_bit(((4*32+ 3)), ((unsigned long *)((c)->x86_capability))) : variable_test_bit(((4*32+ 3)), ((unsigned long *)((c)->x86_capability))))))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/acpi.h", .line = 170, }; ______r = !!((!(__builtin_constant_p((4*32+ 3)) && ( ((((4*32+ 3))>>5)==0 && (1UL<<(((4*32+ 3))&31) & ((1<<((0*32+ 0) & 31))|0|0|(1<<((0*32+ 6) & 31))| (1<<((0*32+ 8) & 31))|0|0|(1<<((0*32+15) & 31))| 0|0))) || ((((4*32+ 3))>>5)==1 && (1UL<<(((4*32+ 3))&31) & (0|0))) || ((((4*32+ 3))>>5)==2 && (1UL<<(((4*32+ 3))&31) & 0)) || ((((4*32+ 3))>>5)==3 && (1UL<<(((4*32+ 3))&31) & (0))) || ((((4*32+ 3))>>5)==4 && (1UL<<(((4*32+ 3))&31) & 0)) || ((((4*32+ 3))>>5)==5 && (1UL<<(((4*32+ 3))&31) & 0)) || ((((4*32+ 3))>>5)==6 && (1UL<<(((4*32+ 3))&31) & 0)) || ((((4*32+ 3))>>5)==7 && (1UL<<(((4*32+ 3))&31) & 0)) || ((((4*32+ 3))>>5)==8 && (1UL<<(((4*32+ 3))&31) & 0)) || ((((4*32+ 3))>>5)==9 && (1UL<<(((4*32+ 3))&31) & 0)) ) ? 1 : (__builtin_constant_p(((4*32+ 3))) ? constant_test_bit(((4*32+ 3)), ((unsigned long *)((c)->x86_capability))) : variable_test_bit(((4*32+ 3)), ((unsigned long *)((c)->x86_capability))))))); ______f.miss_hit[______r]++; ______r; })) buf[2] &= ~((0x0200)); } extern unsigned long __FIXADDR_TOP; enum fixed_addresses { FIX_HOLE, FIX_VDSO, FIX_DBGP_BASE, FIX_EARLYCON_MEM_BASE, FIX_APIC_BASE, FIX_IO_APIC_BASE_0, FIX_IO_APIC_BASE_END = FIX_IO_APIC_BASE_0 + 64 - 1, FIX_KMAP_BEGIN, FIX_KMAP_END = FIX_KMAP_BEGIN+(KM_TYPE_NR*8)-1, FIX_PCIE_MCFG, FIX_PARAVIRT_BOOTMAP, FIX_TEXT_POKE1, FIX_TEXT_POKE0, __end_of_permanent_fixed_addresses, FIX_BTMAP_END = (__end_of_permanent_fixed_addresses ^ (__end_of_permanent_fixed_addresses + (64 * 4) - 1)) & -512 ? __end_of_permanent_fixed_addresses + (64 * 4) - (__end_of_permanent_fixed_addresses & ((64 * 4) - 1)) : __end_of_permanent_fixed_addresses, FIX_BTMAP_BEGIN = FIX_BTMAP_END + (64 * 4) - 1, FIX_WP_TEST, __end_of_fixed_addresses }; extern void reserve_top_address(unsigned long reserve); extern int fixmaps_set; extern pte_t *kmap_pte; extern pgprot_t kmap_prot; extern pte_t *pkmap_page_table; void __native_set_fixmap(enum fixed_addresses idx, pte_t pte); void native_set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t flags); extern void __this_fixmap_does_not_exist(void); static inline __attribute__((always_inline)) __attribute__((always_inline)) unsigned long fix_to_virt(const unsigned int idx) { if (__builtin_constant_p(((idx >= __end_of_fixed_addresses))) ? !!((idx >= __end_of_fixed_addresses)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/fixmap.h", .line = 210, }; ______r = !!((idx >= __end_of_fixed_addresses)); ______f.miss_hit[______r]++; ______r; })) __this_fixmap_does_not_exist(); return (((unsigned long)__FIXADDR_TOP) - ((idx) << 12)); } static inline __attribute__((always_inline)) unsigned long virt_to_fix(const unsigned long vaddr) { do { if (__builtin_constant_p((((__builtin_constant_p(vaddr >= ((unsigned long)__FIXADDR_TOP) || vaddr < (((unsigned long)__FIXADDR_TOP) - (__end_of_permanent_fixed_addresses << 12))) ? !!(vaddr >= ((unsigned long)__FIXADDR_TOP) || vaddr < (((unsigned long)__FIXADDR_TOP) - (__end_of_permanent_fixed_addresses << 12))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/fixmap.h", .line = 218, }; ______r = __builtin_expect(!!(vaddr >= ((unsigned long)__FIXADDR_TOP) || vaddr < (((unsigned long)__FIXADDR_TOP) - (__end_of_permanent_fixed_addresses << 12))), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(vaddr >= ((unsigned long)__FIXADDR_TOP) || vaddr < (((unsigned long)__FIXADDR_TOP) - (__end_of_permanent_fixed_addresses << 12))) ? !!(vaddr >= ((unsigned long)__FIXADDR_TOP) || vaddr < (((unsigned long)__FIXADDR_TOP) - (__end_of_permanent_fixed_addresses << 12))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/fixmap.h", .line = 218, }; ______r = __builtin_expect(!!(vaddr >= ((unsigned long)__FIXADDR_TOP) || vaddr < (((unsigned long)__FIXADDR_TOP) - (__end_of_permanent_fixed_addresses << 12))), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/fixmap.h", .line = 218, }; ______r = !!(((__builtin_constant_p(vaddr >= ((unsigned long)__FIXADDR_TOP) || vaddr < (((unsigned long)__FIXADDR_TOP) - (__end_of_permanent_fixed_addresses << 12))) ? !!(vaddr >= ((unsigned long)__FIXADDR_TOP) || vaddr < (((unsigned long)__FIXADDR_TOP) - (__end_of_permanent_fixed_addresses << 12))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/fixmap.h", .line = 218, }; ______r = __builtin_expect(!!(vaddr >= ((unsigned long)__FIXADDR_TOP) || vaddr < (((unsigned long)__FIXADDR_TOP) - (__end_of_permanent_fixed_addresses << 12))), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) do { asm volatile("1:\tud2\n" ".pushsection __bug_table,\"a\"\n" "2:\t.long 1b, %c0\n" "\t.word %c1, 0\n" "\t.org 2b+%c2\n" ".popsection" : : "i" ("/data/exp/linux-3.0.4/arch/x86/include/asm/fixmap.h"), "i" (218), "i" (sizeof(struct bug_entry))); __builtin_unreachable(); } while (0); } while(0); return ((((unsigned long)__FIXADDR_TOP) - ((vaddr)&(~(((1UL) << 12)-1)))) >> 12); } static inline __attribute__((always_inline)) __attribute__((always_inline)) unsigned long __set_fixmap_offset(enum fixed_addresses idx, phys_addr_t phys, pgprot_t flags) { __set_fixmap(idx, phys, flags); return fix_to_virt(idx) + (phys & (((1UL) << 12) - 1)); } extern void generic_apic_probe(void); extern unsigned int apic_verbosity; extern int local_apic_timer_c2_ok; extern int disable_apic; extern void __inquire_remote_apic(int apicid); static inline __attribute__((always_inline)) void default_inquire_remote_apic(int apicid) { if (__builtin_constant_p(((apic_verbosity >= 2))) ? !!((apic_verbosity >= 2)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/apic.h", .line = 63, }; ______r = !!((apic_verbosity >= 2)); ______f.miss_hit[______r]++; ______r; })) __inquire_remote_apic(apicid); } static inline __attribute__((always_inline)) bool apic_from_smp_config(void) { return smp_found_config && !disable_apic; } static inline __attribute__((always_inline)) int is_vsmp_box(void) { return 0; } extern void xapic_wait_icr_idle(void); extern u32 safe_xapic_wait_icr_idle(void); extern void xapic_icr_write(u32, u32); extern int setup_profiling_timer(unsigned int); static inline __attribute__((always_inline)) void native_apic_mem_write(u32 reg, u32 v) { volatile u32 *addr = (volatile u32 *)((fix_to_virt(FIX_APIC_BASE)) + reg); asm volatile ("661:\n\t" "movl %0, %1" "\n662:\n" ".section .altinstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "661b\n" " " ".long" " " "663f\n" " .word " "(3*32+19)" "\n" " .byte 662b-661b\n" " .byte 664f-663f\n" ".previous\n" ".section .discard,\"aw\",@progbits\n" " .byte 0xff + (664f-663f) - (662b-661b)\n" ".previous\n" ".section .altinstr_replacement, \"ax\"\n" "663:\n\t" "xchgl %0, %1" "\n664:\n" ".previous" : "=r" (v), "=m" (*addr) : "i" (0), "0" (v), "m" (*addr)) ; } static inline __attribute__((always_inline)) u32 native_apic_mem_read(u32 reg) { return *((volatile u32 *)((fix_to_virt(FIX_APIC_BASE)) + reg)); } extern void native_apic_wait_icr_idle(void); extern u32 native_safe_apic_wait_icr_idle(void); extern void native_apic_icr_write(u32 low, u32 id); extern u64 native_apic_icr_read(void); extern int x2apic_mode; static inline __attribute__((always_inline)) void check_x2apic(void) { } static inline __attribute__((always_inline)) void enable_x2apic(void) { } static inline __attribute__((always_inline)) int x2apic_enabled(void) { return 0; } static inline __attribute__((always_inline)) void x2apic_force_phys(void) { } extern void enable_IR_x2apic(void); extern int get_physical_broadcast(void); extern int lapic_get_maxlvt(void); extern void clear_local_APIC(void); extern void connect_bsp_APIC(void); extern void disconnect_bsp_APIC(int virt_wire_setup); extern void disable_local_APIC(void); extern void lapic_shutdown(void); extern int verify_local_APIC(void); extern void sync_Arb_IDs(void); extern void init_bsp_APIC(void); extern void setup_local_APIC(void); extern void end_local_APIC_setup(void); extern void bsp_end_local_APIC_setup(void); extern void init_apic_mappings(void); void register_lapic_address(unsigned long address); extern void setup_boot_APIC_clock(void); extern void setup_secondary_APIC_clock(void); extern int APIC_init_uniprocessor(void); extern int apic_force_enable(unsigned long addr); static inline __attribute__((always_inline)) int apic_is_clustered_box(void) { return 0; } extern int setup_APIC_eilvt(u8 lvt_off, u8 vector, u8 msg_type, u8 mask); struct apic { char *name; int (*probe)(void); int (*acpi_madt_oem_check)(char *oem_id, char *oem_table_id); int (*apic_id_registered)(void); u32 irq_delivery_mode; u32 irq_dest_mode; const struct cpumask *(*target_cpus)(void); int disable_esr; int dest_logical; unsigned long (*check_apicid_used)(physid_mask_t *map, int apicid); unsigned long (*check_apicid_present)(int apicid); void (*vector_allocation_domain)(int cpu, struct cpumask *retmask); void (*init_apic_ldr)(void); void (*ioapic_phys_id_map)(physid_mask_t *phys_map, physid_mask_t *retmap); void (*setup_apic_routing)(void); int (*multi_timer_check)(int apic, int irq); int (*cpu_present_to_apicid)(int mps_cpu); void (*apicid_to_cpu_present)(int phys_apicid, physid_mask_t *retmap); void (*setup_portio_remap)(void); int (*check_phys_apicid_present)(int phys_apicid); void (*enable_apic_mode)(void); int (*phys_pkg_id)(int cpuid_apic, int index_msb); int (*mps_oem_check)(struct mpc_table *mpc, char *oem, char *productid); unsigned int (*get_apic_id)(unsigned long x); unsigned long (*set_apic_id)(unsigned int id); unsigned long apic_id_mask; unsigned int (*cpu_mask_to_apicid)(const struct cpumask *cpumask); unsigned int (*cpu_mask_to_apicid_and)(const struct cpumask *cpumask, const struct cpumask *andmask); void (*send_IPI_mask)(const struct cpumask *mask, int vector); void (*send_IPI_mask_allbutself)(const struct cpumask *mask, int vector); void (*send_IPI_allbutself)(int vector); void (*send_IPI_all)(int vector); void (*send_IPI_self)(int vector); int (*wakeup_secondary_cpu)(int apicid, unsigned long start_eip); int trampoline_phys_low; int trampoline_phys_high; void (*wait_for_init_deassert)(atomic_t *deassert); void (*smp_callin_clear_local_apic)(void); void (*inquire_remote_apic)(int apicid); u32 (*read)(u32 reg); void (*write)(u32 reg, u32 v); u64 (*icr_read)(void); void (*icr_write)(u32 low, u32 high); void (*wait_icr_idle)(void); u32 (*safe_wait_icr_idle)(void); int (*x86_32_early_logical_apicid)(int cpu); int (*x86_32_numa_cpu_node)(int cpu); }; extern struct apic *apic; extern struct apic *__apicdrivers[], *__apicdrivers_end[]; extern atomic_t init_deasserted; extern int wakeup_secondary_cpu_via_nmi(int apicid, unsigned long start_eip); static inline __attribute__((always_inline)) u32 apic_read(u32 reg) { return apic->read(reg); } static inline __attribute__((always_inline)) void apic_write(u32 reg, u32 val) { apic->write(reg, val); } static inline __attribute__((always_inline)) u64 apic_icr_read(void) { return apic->icr_read(); } static inline __attribute__((always_inline)) void apic_icr_write(u32 low, u32 high) { apic->icr_write(low, high); } static inline __attribute__((always_inline)) void apic_wait_icr_idle(void) { apic->wait_icr_idle(); } static inline __attribute__((always_inline)) u32 safe_apic_wait_icr_idle(void) { return apic->safe_wait_icr_idle(); } static inline __attribute__((always_inline)) void ack_APIC_irq(void) { apic_write(0xB0, 0); } static inline __attribute__((always_inline)) unsigned default_get_apic_id(unsigned long x) { unsigned int ver = ((apic_read(0x30)) & 0xFFu); if (__builtin_constant_p(((((ver) >= 0x14) || (__builtin_constant_p((3*32+26)) && ( ((((3*32+26))>>5)==0 && (1UL<<(((3*32+26))&31) & ((1<<((0*32+ 0) & 31))|0|0|(1<<((0*32+ 6) & 31))| (1<<((0*32+ 8) & 31))|0|0|(1<<((0*32+15) & 31))| 0|0))) || ((((3*32+26))>>5)==1 && (1UL<<(((3*32+26))&31) & (0|0))) || ((((3*32+26))>>5)==2 && (1UL<<(((3*32+26))&31) & 0)) || ((((3*32+26))>>5)==3 && (1UL<<(((3*32+26))&31) & (0))) || ((((3*32+26))>>5)==4 && (1UL<<(((3*32+26))&31) & 0)) || ((((3*32+26))>>5)==5 && (1UL<<(((3*32+26))&31) & 0)) || ((((3*32+26))>>5)==6 && (1UL<<(((3*32+26))&31) & 0)) || ((((3*32+26))>>5)==7 && (1UL<<(((3*32+26))&31) & 0)) || ((((3*32+26))>>5)==8 && (1UL<<(((3*32+26))&31) & 0)) || ((((3*32+26))>>5)==9 && (1UL<<(((3*32+26))&31) & 0)) ) ? 1 : (__builtin_constant_p(((3*32+26))) ? constant_test_bit(((3*32+26)), ((unsigned long *)((&boot_cpu_data)->x86_capability))) : variable_test_bit(((3*32+26)), ((unsigned long *)((&boot_cpu_data)->x86_capability)))))))) ? !!((((ver) >= 0x14) || (__builtin_constant_p((3*32+26)) && ( ((((3*32+26))>>5)==0 && (1UL<<(((3*32+26))&31) & ((1<<((0*32+ 0) & 31))|0|0|(1<<((0*32+ 6) & 31))| (1<<((0*32+ 8) & 31))|0|0|(1<<((0*32+15) & 31))| 0|0))) || ((((3*32+26))>>5)==1 && (1UL<<(((3*32+26))&31) & (0|0))) || ((((3*32+26))>>5)==2 && (1UL<<(((3*32+26))&31) & 0)) || ((((3*32+26))>>5)==3 && (1UL<<(((3*32+26))&31) & (0))) || ((((3*32+26))>>5)==4 && (1UL<<(((3*32+26))&31) & 0)) || ((((3*32+26))>>5)==5 && (1UL<<(((3*32+26))&31) & 0)) || ((((3*32+26))>>5)==6 && (1UL<<(((3*32+26))&31) & 0)) || ((((3*32+26))>>5)==7 && (1UL<<(((3*32+26))&31) & 0)) || ((((3*32+26))>>5)==8 && (1UL<<(((3*32+26))&31) & 0)) || ((((3*32+26))>>5)==9 && (1UL<<(((3*32+26))&31) & 0)) ) ? 1 : (__builtin_constant_p(((3*32+26))) ? constant_test_bit(((3*32+26)), ((unsigned long *)((&boot_cpu_data)->x86_capability))) : variable_test_bit(((3*32+26)), ((unsigned long *)((&boot_cpu_data)->x86_capability))))))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/apic.h", .line = 468, }; ______r = !!((((ver) >= 0x14) || (__builtin_constant_p((3*32+26)) && ( ((((3*32+26))>>5)==0 && (1UL<<(((3*32+26))&31) & ((1<<((0*32+ 0) & 31))|0|0|(1<<((0*32+ 6) & 31))| (1<<((0*32+ 8) & 31))|0|0|(1<<((0*32+15) & 31))| 0|0))) || ((((3*32+26))>>5)==1 && (1UL<<(((3*32+26))&31) & (0|0))) || ((((3*32+26))>>5)==2 && (1UL<<(((3*32+26))&31) & 0)) || ((((3*32+26))>>5)==3 && (1UL<<(((3*32+26))&31) & (0))) || ((((3*32+26))>>5)==4 && (1UL<<(((3*32+26))&31) & 0)) || ((((3*32+26))>>5)==5 && (1UL<<(((3*32+26))&31) & 0)) || ((((3*32+26))>>5)==6 && (1UL<<(((3*32+26))&31) & 0)) || ((((3*32+26))>>5)==7 && (1UL<<(((3*32+26))&31) & 0)) || ((((3*32+26))>>5)==8 && (1UL<<(((3*32+26))&31) & 0)) || ((((3*32+26))>>5)==9 && (1UL<<(((3*32+26))&31) & 0)) ) ? 1 : (__builtin_constant_p(((3*32+26))) ? constant_test_bit(((3*32+26)), ((unsigned long *)((&boot_cpu_data)->x86_capability))) : variable_test_bit(((3*32+26)), ((unsigned long *)((&boot_cpu_data)->x86_capability))))))); ______f.miss_hit[______r]++; ______r; })) return (x >> 24) & 0xFF; else return (x >> 24) & 0x0F; } static inline __attribute__((always_inline)) void default_wait_for_init_deassert(atomic_t *deassert) { while (!atomic_read(deassert)) cpu_relax(); return; } extern struct apic *generic_bigsmp_probe(void); static inline __attribute__((always_inline)) const struct cpumask *default_target_cpus(void) { return cpu_online_mask; } extern __attribute__((section(".data..percpu" ""))) __typeof__(u16) x86_bios_cpu_apicid; extern __typeof__(u16) *x86_bios_cpu_apicid_early_ptr; extern __typeof__(u16) x86_bios_cpu_apicid_early_map[]; static inline __attribute__((always_inline)) unsigned int read_apic_id(void) { unsigned int reg; reg = apic_read(0x20); return apic->get_apic_id(reg); } extern void default_setup_apic_routing(void); extern struct apic apic_noop; static inline __attribute__((always_inline)) int noop_x86_32_early_logical_apicid(int cpu) { return 0xFFu; } extern void default_init_apic_ldr(void); static inline __attribute__((always_inline)) int default_apic_id_registered(void) { return (__builtin_constant_p((read_apic_id())) ? constant_test_bit((read_apic_id()), ((phys_cpu_present_map).mask)) : variable_test_bit((read_apic_id()), ((phys_cpu_present_map).mask))); } static inline __attribute__((always_inline)) int default_phys_pkg_id(int cpuid_apic, int index_msb) { return cpuid_apic >> index_msb; } static inline __attribute__((always_inline)) unsigned int default_cpu_mask_to_apicid(const struct cpumask *cpumask) { return ((cpumask)->bits)[0] & 0xFFu; } static inline __attribute__((always_inline)) unsigned int default_cpu_mask_to_apicid_and(const struct cpumask *cpumask, const struct cpumask *andmask) { unsigned long mask1 = ((cpumask)->bits)[0]; unsigned long mask2 = ((andmask)->bits)[0]; unsigned long mask3 = ((cpu_online_mask)->bits)[0]; return (unsigned int)(mask1 & mask2 & mask3); } static inline __attribute__((always_inline)) unsigned long default_check_apicid_used(physid_mask_t *map, int apicid) { return (__builtin_constant_p((apicid)) ? constant_test_bit((apicid), ((*map).mask)) : variable_test_bit((apicid), ((*map).mask))); } static inline __attribute__((always_inline)) unsigned long default_check_apicid_present(int bit) { return (__builtin_constant_p((bit)) ? constant_test_bit((bit), ((phys_cpu_present_map).mask)) : variable_test_bit((bit), ((phys_cpu_present_map).mask))); } static inline __attribute__((always_inline)) void default_ioapic_phys_id_map(physid_mask_t *phys_map, physid_mask_t *retmap) { *retmap = *phys_map; } static inline __attribute__((always_inline)) int __default_cpu_present_to_apicid(int mps_cpu) { if (__builtin_constant_p(((mps_cpu < nr_cpu_ids && (__builtin_constant_p((cpumask_check((mps_cpu)))) ? constant_test_bit((cpumask_check((mps_cpu))), ((((cpu_present_mask))->bits))) : variable_test_bit((cpumask_check((mps_cpu))), ((((cpu_present_mask))->bits))))))) ? !!((mps_cpu < nr_cpu_ids && (__builtin_constant_p((cpumask_check((mps_cpu)))) ? constant_test_bit((cpumask_check((mps_cpu))), ((((cpu_present_mask))->bits))) : variable_test_bit((cpumask_check((mps_cpu))), ((((cpu_present_mask))->bits)))))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/apic.h", .line = 594, }; ______r = !!((mps_cpu < nr_cpu_ids && (__builtin_constant_p((cpumask_check((mps_cpu)))) ? constant_test_bit((cpumask_check((mps_cpu))), ((((cpu_present_mask))->bits))) : variable_test_bit((cpumask_check((mps_cpu))), ((((cpu_present_mask))->bits)))))); ______f.miss_hit[______r]++; ______r; })) return (int)(*({ do { const void *__vpp_verify = (typeof((&(x86_bios_cpu_apicid))))((void *)0); (void)__vpp_verify; } while (0); ({ unsigned long __ptr; __asm__ ("" : "=r"(__ptr) : "0"((typeof(*(&(x86_bios_cpu_apicid))) *)(&(x86_bios_cpu_apicid)))); (typeof((typeof(*(&(x86_bios_cpu_apicid))) *)(&(x86_bios_cpu_apicid)))) (__ptr + (((__per_cpu_offset[mps_cpu])))); }); })); else return 0xFFu; } static inline __attribute__((always_inline)) int __default_check_phys_apicid_present(int phys_apicid) { return (__builtin_constant_p((phys_apicid)) ? constant_test_bit((phys_apicid), ((phys_cpu_present_map).mask)) : variable_test_bit((phys_apicid), ((phys_cpu_present_map).mask))); } static inline __attribute__((always_inline)) int default_cpu_present_to_apicid(int mps_cpu) { return __default_cpu_present_to_apicid(mps_cpu); } static inline __attribute__((always_inline)) int default_check_phys_apicid_present(int phys_apicid) { return __default_check_phys_apicid_present(phys_apicid); } static inline __attribute__((always_inline)) int invalid_vm86_irq(int irq) { return irq < 3 || irq > 15; } union IO_APIC_reg_00 { u32 raw; struct { u32 __reserved_2 : 14, LTS : 1, delivery_type : 1, __reserved_1 : 8, ID : 8; } __attribute__ ((packed)) bits; }; union IO_APIC_reg_01 { u32 raw; struct { u32 version : 8, __reserved_2 : 7, PRQ : 1, entries : 8, __reserved_1 : 8; } __attribute__ ((packed)) bits; }; union IO_APIC_reg_02 { u32 raw; struct { u32 __reserved_2 : 24, arbitration : 4, __reserved_1 : 4; } __attribute__ ((packed)) bits; }; union IO_APIC_reg_03 { u32 raw; struct { u32 boot_DT : 1, __reserved_1 : 31; } __attribute__ ((packed)) bits; }; struct IO_APIC_route_entry { __u32 vector : 8, delivery_mode : 3, dest_mode : 1, delivery_status : 1, polarity : 1, irr : 1, trigger : 1, mask : 1, __reserved_2 : 15; __u32 __reserved_3 : 24, dest : 8; } __attribute__ ((packed)); struct IR_IO_APIC_route_entry { __u64 vector : 8, zero : 3, index2 : 1, delivery_status : 1, polarity : 1, irr : 1, trigger : 1, mask : 1, reserved : 31, format : 1, index : 15; } __attribute__ ((packed)); extern int nr_ioapics; extern int mpc_ioapic_id(int ioapic); extern unsigned int mpc_ioapic_addr(int ioapic); extern struct mp_ioapic_gsi *mp_ioapic_gsi_routing(int ioapic); extern int mp_irq_entries; extern struct mpc_intsrc mp_irqs[256]; extern int mpc_default_type; extern int sis_apic_bug; extern int skip_ioapic_setup; extern int noioapicquirk; extern int noioapicreroute; extern int timer_through_8259; struct io_apic_irq_attr; extern int io_apic_set_pci_routing(struct device *dev, int irq, struct io_apic_irq_attr *irq_attr); void setup_IO_APIC_irq_extra(u32 gsi); extern void ioapic_and_gsi_init(void); extern void ioapic_insert_resources(void); int io_apic_setup_irq_pin_once(unsigned int irq, int node, struct io_apic_irq_attr *attr); extern int save_ioapic_entries(void); extern void mask_ioapic_entries(void); extern int restore_ioapic_entries(void); extern int get_nr_irqs_gsi(void); extern void setup_ioapic_ids_from_mpc(void); extern void setup_ioapic_ids_from_mpc_nocheck(void); struct mp_ioapic_gsi{ u32 gsi_base; u32 gsi_end; }; extern struct mp_ioapic_gsi mp_gsi_routing[]; extern u32 gsi_top; int mp_find_ioapic(u32 gsi); int mp_find_ioapic_pin(int ioapic, u32 gsi); void __attribute__ ((__section__(".init.text"))) __attribute__((__cold__)) __attribute__((no_instrument_function)) mp_register_ioapic(int id, u32 address, u32 gsi_base); extern void __attribute__ ((__section__(".init.text"))) __attribute__((__cold__)) __attribute__((no_instrument_function)) pre_init_apic_IRQ0(void); extern void mp_save_irq(struct mpc_intsrc *m); extern void disable_ioapic_support(void); extern int smp_num_siblings; extern unsigned int num_processors; static inline __attribute__((always_inline)) bool cpu_has_ht_siblings(void) { bool has_siblings = false; has_siblings = (__builtin_constant_p((0*32+28)) && ( ((((0*32+28))>>5)==0 && (1UL<<(((0*32+28))&31) & ((1<<((0*32+ 0) & 31))|0|0|(1<<((0*32+ 6) & 31))| (1<<((0*32+ 8) & 31))|0|0|(1<<((0*32+15) & 31))| 0|0))) || ((((0*32+28))>>5)==1 && (1UL<<(((0*32+28))&31) & (0|0))) || ((((0*32+28))>>5)==2 && (1UL<<(((0*32+28))&31) & 0)) || ((((0*32+28))>>5)==3 && (1UL<<(((0*32+28))&31) & (0))) || ((((0*32+28))>>5)==4 && (1UL<<(((0*32+28))&31) & 0)) || ((((0*32+28))>>5)==5 && (1UL<<(((0*32+28))&31) & 0)) || ((((0*32+28))>>5)==6 && (1UL<<(((0*32+28))&31) & 0)) || ((((0*32+28))>>5)==7 && (1UL<<(((0*32+28))&31) & 0)) || ((((0*32+28))>>5)==8 && (1UL<<(((0*32+28))&31) & 0)) || ((((0*32+28))>>5)==9 && (1UL<<(((0*32+28))&31) & 0)) ) ? 1 : (__builtin_constant_p(((0*32+28))) ? constant_test_bit(((0*32+28)), ((unsigned long *)((&boot_cpu_data)->x86_capability))) : variable_test_bit(((0*32+28)), ((unsigned long *)((&boot_cpu_data)->x86_capability))))) && smp_num_siblings > 1; return has_siblings; } extern __attribute__((section(".data..percpu" ""))) __typeof__(cpumask_var_t) cpu_sibling_map; extern __attribute__((section(".data..percpu" ""))) __typeof__(cpumask_var_t) cpu_core_map; extern __attribute__((section(".data..percpu" ""))) __typeof__(cpumask_var_t) cpu_llc_shared_map; extern __attribute__((section(".data..percpu" ""))) __typeof__(u16) cpu_llc_id; extern __attribute__((section(".data..percpu" ""))) __typeof__(int) cpu_number; static inline __attribute__((always_inline)) struct cpumask *cpu_sibling_mask(int cpu) { return (*({ do { const void *__vpp_verify = (typeof((&(cpu_sibling_map))))((void *)0); (void)__vpp_verify; } while (0); ({ unsigned long __ptr; __asm__ ("" : "=r"(__ptr) : "0"((typeof(*(&(cpu_sibling_map))) *)(&(cpu_sibling_map)))); (typeof((typeof(*(&(cpu_sibling_map))) *)(&(cpu_sibling_map)))) (__ptr + (((__per_cpu_offset[cpu])))); }); })); } static inline __attribute__((always_inline)) struct cpumask *cpu_core_mask(int cpu) { return (*({ do { const void *__vpp_verify = (typeof((&(cpu_core_map))))((void *)0); (void)__vpp_verify; } while (0); ({ unsigned long __ptr; __asm__ ("" : "=r"(__ptr) : "0"((typeof(*(&(cpu_core_map))) *)(&(cpu_core_map)))); (typeof((typeof(*(&(cpu_core_map))) *)(&(cpu_core_map)))) (__ptr + (((__per_cpu_offset[cpu])))); }); })); } static inline __attribute__((always_inline)) struct cpumask *cpu_llc_shared_mask(int cpu) { return (*({ do { const void *__vpp_verify = (typeof((&(cpu_llc_shared_map))))((void *)0); (void)__vpp_verify; } while (0); ({ unsigned long __ptr; __asm__ ("" : "=r"(__ptr) : "0"((typeof(*(&(cpu_llc_shared_map))) *)(&(cpu_llc_shared_map)))); (typeof((typeof(*(&(cpu_llc_shared_map))) *)(&(cpu_llc_shared_map)))) (__ptr + (((__per_cpu_offset[cpu])))); }); })); } extern __attribute__((section(".data..percpu" ""))) __typeof__(u16) x86_cpu_to_apicid; extern __typeof__(u16) *x86_cpu_to_apicid_early_ptr; extern __typeof__(u16) x86_cpu_to_apicid_early_map[]; extern __attribute__((section(".data..percpu" ""))) __typeof__(u16) x86_bios_cpu_apicid; extern __typeof__(u16) *x86_bios_cpu_apicid_early_ptr; extern __typeof__(u16) x86_bios_cpu_apicid_early_map[]; extern __attribute__((section(".data..percpu" ""))) __typeof__(int) x86_cpu_to_logical_apicid; extern __typeof__(int) *x86_cpu_to_logical_apicid_early_ptr; extern __typeof__(int) x86_cpu_to_logical_apicid_early_map[]; extern unsigned long stack_start; struct smp_ops { void (*smp_prepare_boot_cpu)(void); void (*smp_prepare_cpus)(unsigned max_cpus); void (*smp_cpus_done)(unsigned max_cpus); void (*stop_other_cpus)(int wait); void (*smp_send_reschedule)(int cpu); int (*cpu_up)(unsigned cpu); int (*cpu_disable)(void); void (*cpu_die)(unsigned int cpu); void (*play_dead)(void); void (*send_call_func_ipi)(const struct cpumask *mask); void (*send_call_func_single_ipi)(int cpu); }; extern void set_cpu_sibling_map(int cpu); extern struct smp_ops smp_ops; static inline __attribute__((always_inline)) void smp_send_stop(void) { smp_ops.stop_other_cpus(0); } static inline __attribute__((always_inline)) void stop_other_cpus(void) { smp_ops.stop_other_cpus(1); } static inline __attribute__((always_inline)) void smp_prepare_boot_cpu(void) { smp_ops.smp_prepare_boot_cpu(); } static inline __attribute__((always_inline)) void smp_prepare_cpus(unsigned int max_cpus) { smp_ops.smp_prepare_cpus(max_cpus); } static inline __attribute__((always_inline)) void smp_cpus_done(unsigned int max_cpus) { smp_ops.smp_cpus_done(max_cpus); } static inline __attribute__((always_inline)) int __cpu_up(unsigned int cpu) { return smp_ops.cpu_up(cpu); } static inline __attribute__((always_inline)) int __cpu_disable(void) { return smp_ops.cpu_disable(); } static inline __attribute__((always_inline)) void __cpu_die(unsigned int cpu) { smp_ops.cpu_die(cpu); } static inline __attribute__((always_inline)) void play_dead(void) { smp_ops.play_dead(); } static inline __attribute__((always_inline)) void smp_send_reschedule(int cpu) { smp_ops.smp_send_reschedule(cpu); } static inline __attribute__((always_inline)) void arch_send_call_function_single_ipi(int cpu) { smp_ops.send_call_func_single_ipi(cpu); } static inline __attribute__((always_inline)) void arch_send_call_function_ipi_mask(const struct cpumask *mask) { smp_ops.send_call_func_ipi(mask); } void cpu_disable_common(void); void native_smp_prepare_boot_cpu(void); void native_smp_prepare_cpus(unsigned int max_cpus); void native_smp_cpus_done(unsigned int max_cpus); int native_cpu_up(unsigned int cpunum); int native_cpu_disable(void); void native_cpu_die(unsigned int cpu); void native_play_dead(void); void play_dead_common(void); void wbinvd_on_cpu(int cpu); int wbinvd_on_all_cpus(void); void native_send_call_func_ipi(const struct cpumask *mask); void native_send_call_func_single_ipi(int cpu); void smp_store_cpu_info(int id); static inline __attribute__((always_inline)) int num_booting_cpus(void) { return cpumask_weight(cpu_callout_mask); } extern unsigned disabled_cpus __attribute__ ((__section__(".cpuinit.data"))); extern int safe_smp_processor_id(void); static inline __attribute__((always_inline)) int logical_smp_processor_id(void) { return (((apic_read(0xD0)) >> 24) & 0xFFu); } extern int hard_smp_processor_id(void); extern void smp_send_stop(void); extern void smp_send_reschedule(int cpu); extern void smp_prepare_cpus(unsigned int max_cpus); extern int __cpu_up(unsigned int cpunum); extern void smp_cpus_done(unsigned int max_cpus); int smp_call_function(smp_call_func_t func, void *info, int wait); void smp_call_function_many(const struct cpumask *mask, smp_call_func_t func, void *info, bool wait); void __smp_call_function_single(int cpuid, struct call_single_data *data, int wait); int smp_call_function_any(const struct cpumask *mask, smp_call_func_t func, void *info, int wait); void __attribute__ ((__section__(".init.text"))) __attribute__((__cold__)) __attribute__((no_instrument_function)) call_function_init(void); void generic_smp_call_function_single_interrupt(void); void generic_smp_call_function_interrupt(void); void ipi_call_lock(void); void ipi_call_unlock(void); void ipi_call_lock_irq(void); void ipi_call_unlock_irq(void); int on_each_cpu(smp_call_func_t func, void *info, int wait); void smp_prepare_boot_cpu(void); extern unsigned int setup_max_cpus; extern void __attribute__ ((__section__(".init.text"))) __attribute__((__cold__)) __attribute__((no_instrument_function)) setup_nr_cpu_ids(void); extern void __attribute__ ((__section__(".init.text"))) __attribute__((__cold__)) __attribute__((no_instrument_function)) smp_init(void); extern unsigned int debug_smp_processor_id(void); extern void arch_disable_smp_support(void); void smp_setup_processor_id(void); enum pageblock_bits { PB_migrate, PB_migrate_end = PB_migrate + 3 - 1, NR_PAGEBLOCK_BITS }; struct page; unsigned long get_pageblock_flags_group(struct page *page, int start_bitidx, int end_bitidx); void set_pageblock_flags_group(struct page *page, unsigned long flags, int start_bitidx, int end_bitidx); extern int page_group_by_mobility_disabled; static inline __attribute__((always_inline)) int get_pageblock_migratetype(struct page *page) { return get_pageblock_flags_group(page, PB_migrate, PB_migrate_end); } struct free_area { struct list_head free_list[5]; unsigned long nr_free; }; struct pglist_data; struct zone_padding { char x[0]; } __attribute__((__aligned__(1 << (6)))); enum zone_stat_item { NR_FREE_PAGES, NR_LRU_BASE, NR_INACTIVE_ANON = NR_LRU_BASE, NR_ACTIVE_ANON, NR_INACTIVE_FILE, NR_ACTIVE_FILE, NR_UNEVICTABLE, NR_MLOCK, NR_ANON_PAGES, NR_FILE_MAPPED, NR_FILE_PAGES, NR_FILE_DIRTY, NR_WRITEBACK, NR_SLAB_RECLAIMABLE, NR_SLAB_UNRECLAIMABLE, NR_PAGETABLE, NR_KERNEL_STACK, NR_UNSTABLE_NFS, NR_BOUNCE, NR_VMSCAN_WRITE, NR_WRITEBACK_TEMP, NR_ISOLATED_ANON, NR_ISOLATED_FILE, NR_SHMEM, NR_DIRTIED, NR_WRITTEN, NR_ANON_TRANSPARENT_HUGEPAGES, NR_VM_ZONE_STAT_ITEMS }; enum lru_list { LRU_INACTIVE_ANON = 0, LRU_ACTIVE_ANON = 0 + 1, LRU_INACTIVE_FILE = 0 + 2, LRU_ACTIVE_FILE = 0 + 2 + 1, LRU_UNEVICTABLE, NR_LRU_LISTS }; static inline __attribute__((always_inline)) int is_file_lru(enum lru_list l) { return (l == LRU_INACTIVE_FILE || l == LRU_ACTIVE_FILE); } static inline __attribute__((always_inline)) int is_active_lru(enum lru_list l) { return (l == LRU_ACTIVE_ANON || l == LRU_ACTIVE_FILE); } static inline __attribute__((always_inline)) int is_unevictable_lru(enum lru_list l) { return (l == LRU_UNEVICTABLE); } enum zone_watermarks { WMARK_MIN, WMARK_LOW, WMARK_HIGH, NR_WMARK }; struct per_cpu_pages { int count; int high; int batch; struct list_head lists[3]; }; struct per_cpu_pageset { struct per_cpu_pages pcp; s8 stat_threshold; s8 vm_stat_diff[NR_VM_ZONE_STAT_ITEMS]; }; enum zone_type { ZONE_DMA, ZONE_NORMAL, ZONE_HIGHMEM, ZONE_MOVABLE, __MAX_NR_ZONES }; struct zone_reclaim_stat { unsigned long recent_rotated[2]; unsigned long recent_scanned[2]; }; struct zone { unsigned long watermark[NR_WMARK]; unsigned long percpu_drift_mark; unsigned long lowmem_reserve[4]; struct per_cpu_pageset *pageset; spinlock_t lock; int all_unreclaimable; struct free_area free_area[11]; unsigned long *pageblock_flags; unsigned int compact_considered; unsigned int compact_defer_shift; struct zone_padding _pad1_; spinlock_t lru_lock; struct zone_lru { struct list_head list; } lru[NR_LRU_LISTS]; struct zone_reclaim_stat reclaim_stat; unsigned long pages_scanned; unsigned long flags; atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS]; unsigned int inactive_ratio; struct zone_padding _pad2_; wait_queue_head_t * wait_table; unsigned long wait_table_hash_nr_entries; unsigned long wait_table_bits; struct pglist_data *zone_pgdat; unsigned long zone_start_pfn; unsigned long spanned_pages; unsigned long present_pages; const char *name; } __attribute__((__aligned__(1 << (6)))); typedef enum { ZONE_RECLAIM_LOCKED, ZONE_OOM_LOCKED, ZONE_CONGESTED, } zone_flags_t; static inline __attribute__((always_inline)) void zone_set_flag(struct zone *zone, zone_flags_t flag) { set_bit(flag, &zone->flags); } static inline __attribute__((always_inline)) int zone_test_and_set_flag(struct zone *zone, zone_flags_t flag) { return test_and_set_bit(flag, &zone->flags); } static inline __attribute__((always_inline)) void zone_clear_flag(struct zone *zone, zone_flags_t flag) { clear_bit(flag, &zone->flags); } static inline __attribute__((always_inline)) int zone_is_reclaim_congested(const struct zone *zone) { return (__builtin_constant_p((ZONE_CONGESTED)) ? constant_test_bit((ZONE_CONGESTED), (&zone->flags)) : variable_test_bit((ZONE_CONGESTED), (&zone->flags))); } static inline __attribute__((always_inline)) int zone_is_reclaim_locked(const struct zone *zone) { return (__builtin_constant_p((ZONE_RECLAIM_LOCKED)) ? constant_test_bit((ZONE_RECLAIM_LOCKED), (&zone->flags)) : variable_test_bit((ZONE_RECLAIM_LOCKED), (&zone->flags))); } static inline __attribute__((always_inline)) int zone_is_oom_locked(const struct zone *zone) { return (__builtin_constant_p((ZONE_OOM_LOCKED)) ? constant_test_bit((ZONE_OOM_LOCKED), (&zone->flags)) : variable_test_bit((ZONE_OOM_LOCKED), (&zone->flags))); } struct zonelist_cache; struct zoneref { struct zone *zone; int zone_idx; }; struct zonelist { struct zonelist_cache *zlcache_ptr; struct zoneref _zonerefs[((1 << 0) * 4) + 1]; }; struct node_active_region { unsigned long start_pfn; unsigned long end_pfn; int nid; }; extern struct page *mem_map; struct bootmem_data; typedef struct pglist_data { struct zone node_zones[4]; struct zonelist node_zonelists[1]; int nr_zones; struct page *node_mem_map; unsigned long node_start_pfn; unsigned long node_present_pages; unsigned long node_spanned_pages; int node_id; wait_queue_head_t kswapd_wait; struct task_struct *kswapd; int kswapd_max_order; enum zone_type classzone_idx; } pg_data_t; struct rw_semaphore; struct rw_semaphore { long count; spinlock_t wait_lock; struct list_head wait_list; struct lockdep_map dep_map; }; extern struct rw_semaphore *rwsem_down_read_failed(struct rw_semaphore *sem); extern struct rw_semaphore *rwsem_down_write_failed(struct rw_semaphore *sem); extern struct rw_semaphore *rwsem_wake(struct rw_semaphore *); extern struct rw_semaphore *rwsem_downgrade_wake(struct rw_semaphore *sem); static inline __attribute__((always_inline)) void __down_read(struct rw_semaphore *sem) { asm volatile("# beginning down_read\n\t" ".section .smp_locks,\"a\"\n" ".balign 4\n" ".long 671f - .\n" ".previous\n" "671:" "\n\tlock; " " " "incl" " " "(%1)\n\t" " jns 1f\n" " call call_rwsem_down_read_failed\n" "1:\n\t" "# ending down_read\n\t" : "+m" (sem->count) : "a" (sem) : "memory", "cc"); } static inline __attribute__((always_inline)) int __down_read_trylock(struct rw_semaphore *sem) { long result, tmp; asm volatile("# beginning __down_read_trylock\n\t" " mov %0,%1\n\t" "1:\n\t" " mov %1,%2\n\t" " add %3,%2\n\t" " jle 2f\n\t" ".section .smp_locks,\"a\"\n" ".balign 4\n" ".long 671f - .\n" ".previous\n" "671:" "\n\tlock; " " cmpxchg %2,%0\n\t" " jnz 1b\n\t" "2:\n\t" "# ending __down_read_trylock\n\t" : "+m" (sem->count), "=&a" (result), "=&r" (tmp) : "i" (0x00000001L) : "memory", "cc"); return result >= 0 ? 1 : 0; } static inline __attribute__((always_inline)) void __down_write_nested(struct rw_semaphore *sem, int subclass) { long tmp; asm volatile("# beginning down_write\n\t" ".section .smp_locks,\"a\"\n" ".balign 4\n" ".long 671f - .\n" ".previous\n" "671:" "\n\tlock; " " xadd %1,(%2)\n\t" " test %1,%1\n\t" " jz 1f\n" " call call_rwsem_down_write_failed\n" "1:\n" "# ending down_write" : "+m" (sem->count), "=d" (tmp) : "a" (sem), "1" (((-0x0000ffffL -1) + 0x00000001L)) : "memory", "cc"); } static inline __attribute__((always_inline)) void __down_write(struct rw_semaphore *sem) { __down_write_nested(sem, 0); } static inline __attribute__((always_inline)) int __down_write_trylock(struct rw_semaphore *sem) { long ret = ({ __typeof__(*(((&sem->count)))) __ret; __typeof__(*(((&sem->count)))) __old = (((0x00000000L))); __typeof__(*(((&sem->count)))) __new = (((((-0x0000ffffL -1) + 0x00000001L)))); switch ((sizeof(*&sem->count))) { case 1: { volatile u8 *__ptr = (volatile u8 *)(((&sem->count))); asm volatile(".section .smp_locks,\"a\"\n" ".balign 4\n" ".long 671f - .\n" ".previous\n" "671:" "\n\tlock; " "cmpxchgb %2,%1" : "=a" (__ret), "+m" (*__ptr) : "q" (__new), "0" (__old) : "memory"); break; } case 2: { volatile u16 *__ptr = (volatile u16 *)(((&sem->count))); asm volatile(".section .smp_locks,\"a\"\n" ".balign 4\n" ".long 671f - .\n" ".previous\n" "671:" "\n\tlock; " "cmpxchgw %2,%1" : "=a" (__ret), "+m" (*__ptr) : "r" (__new), "0" (__old) : "memory"); break; } case 4: { volatile u32 *__ptr = (volatile u32 *)(((&sem->count))); asm volatile(".section .smp_locks,\"a\"\n" ".balign 4\n" ".long 671f - .\n" ".previous\n" "671:" "\n\tlock; " "cmpxchgl %2,%1" : "=a" (__ret), "+m" (*__ptr) : "r" (__new), "0" (__old) : "memory"); break; } default: __cmpxchg_wrong_size(); } __ret; }) ; if (__builtin_constant_p(((ret == 0x00000000L))) ? !!((ret == 0x00000000L)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/rwsem.h", .line = 131, }; ______r = !!((ret == 0x00000000L)); ______f.miss_hit[______r]++; ______r; })) return 1; return 0; } static inline __attribute__((always_inline)) void __up_read(struct rw_semaphore *sem) { long tmp; asm volatile("# beginning __up_read\n\t" ".section .smp_locks,\"a\"\n" ".balign 4\n" ".long 671f - .\n" ".previous\n" "671:" "\n\tlock; " " xadd %1,(%2)\n\t" " jns 1f\n\t" " call call_rwsem_wake\n" "1:\n" "# ending __up_read\n" : "+m" (sem->count), "=d" (tmp) : "a" (sem), "1" (-0x00000001L) : "memory", "cc"); } static inline __attribute__((always_inline)) void __up_write(struct rw_semaphore *sem) { long tmp; asm volatile("# beginning __up_write\n\t" ".section .smp_locks,\"a\"\n" ".balign 4\n" ".long 671f - .\n" ".previous\n" "671:" "\n\tlock; " " xadd %1,(%2)\n\t" " jns 1f\n\t" " call call_rwsem_wake\n" "1:\n\t" "# ending __up_write\n" : "+m" (sem->count), "=d" (tmp) : "a" (sem), "1" (-((-0x0000ffffL -1) + 0x00000001L)) : "memory", "cc"); } static inline __attribute__((always_inline)) void __downgrade_write(struct rw_semaphore *sem) { asm volatile("# beginning __downgrade_write\n\t" ".section .smp_locks,\"a\"\n" ".balign 4\n" ".long 671f - .\n" ".previous\n" "671:" "\n\tlock; " " " "addl" " " "%2,(%1)\n\t" " jns 1f\n\t" " call call_rwsem_downgrade_wake\n" "1:\n\t" "# ending __downgrade_write\n" : "+m" (sem->count) : "a" (sem), "er" (-(-0x0000ffffL -1)) : "memory", "cc"); } static inline __attribute__((always_inline)) void rwsem_atomic_add(long delta, struct rw_semaphore *sem) { asm volatile(".section .smp_locks,\"a\"\n" ".balign 4\n" ".long 671f - .\n" ".previous\n" "671:" "\n\tlock; " " " "addl" " " "%1,%0" : "+m" (sem->count) : "er" (delta)); } static inline __attribute__((always_inline)) long rwsem_atomic_update(long delta, struct rw_semaphore *sem) { long tmp = delta; asm volatile(".section .smp_locks,\"a\"\n" ".balign 4\n" ".long 671f - .\n" ".previous\n" "671:" "\n\tlock; " "xadd %0,%1" : "+r" (tmp), "+m" (sem->count) : : "memory"); return tmp + delta; } static inline __attribute__((always_inline)) int rwsem_is_locked(struct rw_semaphore *sem) { return sem->count != 0; } extern void __init_rwsem(struct rw_semaphore *sem, const char *name, struct lock_class_key *key); extern void down_read(struct rw_semaphore *sem); extern int down_read_trylock(struct rw_semaphore *sem); extern void down_write(struct rw_semaphore *sem); extern int down_write_trylock(struct rw_semaphore *sem); extern void up_read(struct rw_semaphore *sem); extern void up_write(struct rw_semaphore *sem); extern void downgrade_write(struct rw_semaphore *sem); extern void down_read_nested(struct rw_semaphore *sem, int subclass); extern void down_write_nested(struct rw_semaphore *sem, int subclass); extern void down_read_non_owner(struct rw_semaphore *sem); extern void up_read_non_owner(struct rw_semaphore *sem); struct srcu_struct_array { int c[2]; }; struct srcu_struct { int completed; struct srcu_struct_array *per_cpu_ref; struct mutex mutex; struct lockdep_map dep_map; }; int __init_srcu_struct(struct srcu_struct *sp, const char *name, struct lock_class_key *key); void cleanup_srcu_struct(struct srcu_struct *sp); int __srcu_read_lock(struct srcu_struct *sp) ; void __srcu_read_unlock(struct srcu_struct *sp, int idx) ; void synchronize_srcu(struct srcu_struct *sp); void synchronize_srcu_expedited(struct srcu_struct *sp); long srcu_batches_completed(struct srcu_struct *sp); static inline __attribute__((always_inline)) int srcu_read_lock_held(struct srcu_struct *sp) { if (__builtin_constant_p(((debug_locks))) ? !!((debug_locks)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/srcu.h", .line = 96, }; ______r = !!((debug_locks)); ______f.miss_hit[______r]++; ______r; })) return lock_is_held(&sp->dep_map); return 1; } static inline __attribute__((always_inline)) int srcu_read_lock(struct srcu_struct *sp) { int retval = __srcu_read_lock(sp); lock_acquire(&(sp)->dep_map, 0, 0, 2, 1, ((void *)0), ({ __label__ __here; __here: (unsigned long)&&__here; })); return retval; } static inline __attribute__((always_inline)) void srcu_read_unlock(struct srcu_struct *sp, int idx) { lock_release(&(sp)->dep_map, 1, ({ __label__ __here; __here: (unsigned long)&&__here; })); __srcu_read_unlock(sp, idx); } struct notifier_block { int (*notifier_call)(struct notifier_block *, unsigned long, void *); struct notifier_block *next; int priority; }; struct atomic_notifier_head { spinlock_t lock; struct notifier_block *head; }; struct blocking_notifier_head { struct rw_semaphore rwsem; struct notifier_block *head; }; struct raw_notifier_head { struct notifier_block *head; }; struct srcu_notifier_head { struct mutex mutex; struct srcu_struct srcu; struct notifier_block *head; }; extern void srcu_init_notifier_head(struct srcu_notifier_head *nh); extern int atomic_notifier_chain_register(struct atomic_notifier_head *nh, struct notifier_block *nb); extern int blocking_notifier_chain_register(struct blocking_notifier_head *nh, struct notifier_block *nb); extern int raw_notifier_chain_register(struct raw_notifier_head *nh, struct notifier_block *nb); extern int srcu_notifier_chain_register(struct srcu_notifier_head *nh, struct notifier_block *nb); extern int blocking_notifier_chain_cond_register( struct blocking_notifier_head *nh, struct notifier_block *nb); extern int atomic_notifier_chain_unregister(struct atomic_notifier_head *nh, struct notifier_block *nb); extern int blocking_notifier_chain_unregister(struct blocking_notifier_head *nh, struct notifier_block *nb); extern int raw_notifier_chain_unregister(struct raw_notifier_head *nh, struct notifier_block *nb); extern int srcu_notifier_chain_unregister(struct srcu_notifier_head *nh, struct notifier_block *nb); extern int atomic_notifier_call_chain(struct atomic_notifier_head *nh, unsigned long val, void *v); extern int __atomic_notifier_call_chain(struct atomic_notifier_head *nh, unsigned long val, void *v, int nr_to_call, int *nr_calls); extern int blocking_notifier_call_chain(struct blocking_notifier_head *nh, unsigned long val, void *v); extern int __blocking_notifier_call_chain(struct blocking_notifier_head *nh, unsigned long val, void *v, int nr_to_call, int *nr_calls); extern int raw_notifier_call_chain(struct raw_notifier_head *nh, unsigned long val, void *v); extern int __raw_notifier_call_chain(struct raw_notifier_head *nh, unsigned long val, void *v, int nr_to_call, int *nr_calls); extern int srcu_notifier_call_chain(struct srcu_notifier_head *nh, unsigned long val, void *v); extern int __srcu_notifier_call_chain(struct srcu_notifier_head *nh, unsigned long val, void *v, int nr_to_call, int *nr_calls); static inline __attribute__((always_inline)) int notifier_from_errno(int err) { if (__builtin_constant_p(((err))) ? !!((err)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/notifier.h", .line = 167, }; ______r = !!((err)); ______f.miss_hit[______r]++; ______r; })) return 0x8000 | (0x0001 - err); return 0x0001; } static inline __attribute__((always_inline)) int notifier_to_errno(int ret) { ret &= ~0x8000; return ret > 0x0001 ? 0x0001 - ret : 0; } extern struct blocking_notifier_head reboot_notifier_list; struct page; struct zone; struct pglist_data; struct mem_section; static inline __attribute__((always_inline)) void pgdat_resize_lock(struct pglist_data *p, unsigned long *f) {} static inline __attribute__((always_inline)) void pgdat_resize_unlock(struct pglist_data *p, unsigned long *f) {} static inline __attribute__((always_inline)) void pgdat_resize_init(struct pglist_data *pgdat) {} static inline __attribute__((always_inline)) unsigned zone_span_seqbegin(struct zone *zone) { return 0; } static inline __attribute__((always_inline)) int zone_span_seqretry(struct zone *zone, unsigned iv) { return 0; } static inline __attribute__((always_inline)) void zone_span_writelock(struct zone *zone) {} static inline __attribute__((always_inline)) void zone_span_writeunlock(struct zone *zone) {} static inline __attribute__((always_inline)) void zone_seqlock_init(struct zone *zone) {} static inline __attribute__((always_inline)) int mhp_notimplemented(const char *func) { printk("<4>" "%s() called, with CONFIG_MEMORY_HOTPLUG disabled\n", func); dump_stack(); return -38; } static inline __attribute__((always_inline)) void register_page_bootmem_info_node(struct pglist_data *pgdat) { } static inline __attribute__((always_inline)) void lock_memory_hotplug(void) {} static inline __attribute__((always_inline)) void unlock_memory_hotplug(void) {} static inline __attribute__((always_inline)) int is_mem_section_removable(unsigned long pfn, unsigned long nr_pages) { return 0; } extern int mem_online_node(int nid); extern int add_memory(int nid, u64 start, u64 size); extern int arch_add_memory(int nid, u64 start, u64 size); extern int remove_memory(u64 start, u64 size); extern int sparse_add_one_section(struct zone *zone, unsigned long start_pfn, int nr_pages); extern void sparse_remove_one_section(struct zone *zone, struct mem_section *ms); extern struct page *sparse_decode_mem_map(unsigned long coded_mem_map, unsigned long pnum); extern struct mutex zonelists_mutex; void build_all_zonelists(void *data); void wakeup_kswapd(struct zone *zone, int order, enum zone_type classzone_idx); bool zone_watermark_ok(struct zone *z, int order, unsigned long mark, int classzone_idx, int alloc_flags); bool zone_watermark_ok_safe(struct zone *z, int order, unsigned long mark, int classzone_idx, int alloc_flags); enum memmap_context { MEMMAP_EARLY, MEMMAP_HOTPLUG, }; extern int init_currently_empty_zone(struct zone *zone, unsigned long start_pfn, unsigned long size, enum memmap_context context); static inline __attribute__((always_inline)) void memory_present(int nid, unsigned long start, unsigned long end) {} static inline __attribute__((always_inline)) int local_memory_node(int node_id) { return node_id; }; static inline __attribute__((always_inline)) int populated_zone(struct zone *zone) { return (!!zone->present_pages); } extern int movable_zone; static inline __attribute__((always_inline)) int zone_movable_is_highmem(void) { return movable_zone == ZONE_HIGHMEM; } static inline __attribute__((always_inline)) int is_highmem_idx(enum zone_type idx) { return (idx == ZONE_HIGHMEM || (idx == ZONE_MOVABLE && zone_movable_is_highmem())); } static inline __attribute__((always_inline)) int is_normal_idx(enum zone_type idx) { return (idx == ZONE_NORMAL); } static inline __attribute__((always_inline)) int is_highmem(struct zone *zone) { int zone_off = (char *)zone - (char *)zone->zone_pgdat->node_zones; return zone_off == ZONE_HIGHMEM * sizeof(*zone) || (zone_off == ZONE_MOVABLE * sizeof(*zone) && zone_movable_is_highmem()); } static inline __attribute__((always_inline)) int is_normal(struct zone *zone) { return zone == zone->zone_pgdat->node_zones + ZONE_NORMAL; } static inline __attribute__((always_inline)) int is_dma32(struct zone *zone) { return 0; } static inline __attribute__((always_inline)) int is_dma(struct zone *zone) { return zone == zone->zone_pgdat->node_zones + ZONE_DMA; } struct ctl_table; int min_free_kbytes_sysctl_handler(struct ctl_table *, int, void *, size_t *, loff_t *); extern int sysctl_lowmem_reserve_ratio[4 -1]; int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *, int, void *, size_t *, loff_t *); int percpu_pagelist_fraction_sysctl_handler(struct ctl_table *, int, void *, size_t *, loff_t *); int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *, int, void *, size_t *, loff_t *); int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *, int, void *, size_t *, loff_t *); extern int numa_zonelist_order_handler(struct ctl_table *, int, void *, size_t *, loff_t *); extern char numa_zonelist_order[]; extern struct pglist_data contig_page_data; extern struct pglist_data *first_online_pgdat(void); extern struct pglist_data *next_online_pgdat(struct pglist_data *pgdat); extern struct zone *next_zone(struct zone *zone); static inline __attribute__((always_inline)) struct zone *zonelist_zone(struct zoneref *zoneref) { return zoneref->zone; } static inline __attribute__((always_inline)) int zonelist_zone_idx(struct zoneref *zoneref) { return zoneref->zone_idx; } static inline __attribute__((always_inline)) int zonelist_node_idx(struct zoneref *zoneref) { return 0; } struct zoneref *next_zones_zonelist(struct zoneref *z, enum zone_type highest_zoneidx, nodemask_t *nodes, struct zone **zone); static inline __attribute__((always_inline)) struct zoneref *first_zones_zonelist(struct zonelist *zonelist, enum zone_type highest_zoneidx, nodemask_t *nodes, struct zone **zone) { return next_zones_zonelist(zonelist->_zonerefs, highest_zoneidx, nodes, zone); } void memory_present(int nid, unsigned long start, unsigned long end); unsigned long __attribute__ ((__section__(".init.text"))) __attribute__((__cold__)) __attribute__((no_instrument_function)) node_memmap_size_bytes(int, unsigned long, unsigned long); static inline __attribute__((always_inline)) int memmap_valid_within(unsigned long pfn, struct page *page, struct zone *zone) { return 1; } extern void *pcpu_base_addr; extern const unsigned long *pcpu_unit_offsets; struct pcpu_group_info { int nr_units; unsigned long base_offset; unsigned int *cpu_map; }; struct pcpu_alloc_info { size_t static_size; size_t reserved_size; size_t dyn_size; size_t unit_size; size_t atom_size; size_t alloc_size; size_t __ai_size; int nr_groups; struct pcpu_group_info groups[]; }; enum pcpu_fc { PCPU_FC_AUTO, PCPU_FC_EMBED, PCPU_FC_PAGE, PCPU_FC_NR, }; extern const char *pcpu_fc_names[PCPU_FC_NR]; extern enum pcpu_fc pcpu_chosen_fc; typedef void * (*pcpu_fc_alloc_fn_t)(unsigned int cpu, size_t size, size_t align); typedef void (*pcpu_fc_free_fn_t)(void *ptr, size_t size); typedef void (*pcpu_fc_populate_pte_fn_t)(unsigned long addr); typedef int (pcpu_fc_cpu_distance_fn_t)(unsigned int from, unsigned int to); extern struct pcpu_alloc_info * __attribute__ ((__section__(".init.text"))) __attribute__((__cold__)) __attribute__((no_instrument_function)) pcpu_alloc_alloc_info(int nr_groups, int nr_units); extern void __attribute__ ((__section__(".init.text"))) __attribute__((__cold__)) __attribute__((no_instrument_function)) pcpu_free_alloc_info(struct pcpu_alloc_info *ai); extern int __attribute__ ((__section__(".init.text"))) __attribute__((__cold__)) __attribute__((no_instrument_function)) pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai, void *base_addr); extern int __attribute__ ((__section__(".init.text"))) __attribute__((__cold__)) __attribute__((no_instrument_function)) pcpu_embed_first_chunk(size_t reserved_size, size_t dyn_size, size_t atom_size, pcpu_fc_cpu_distance_fn_t cpu_distance_fn, pcpu_fc_alloc_fn_t alloc_fn, pcpu_fc_free_fn_t free_fn); extern int __attribute__ ((__section__(".init.text"))) __attribute__((__cold__)) __attribute__((no_instrument_function)) pcpu_page_first_chunk(size_t reserved_size, pcpu_fc_alloc_fn_t alloc_fn, pcpu_fc_free_fn_t free_fn, pcpu_fc_populate_pte_fn_t populate_pte_fn); extern void *__alloc_reserved_percpu(size_t size, size_t align); extern bool is_kernel_percpu_address(unsigned long addr); extern void __attribute__ ((__section__(".init.text"))) __attribute__((__cold__)) __attribute__((no_instrument_function)) percpu_init_late(void); extern void *__alloc_percpu(size_t size, size_t align); extern void free_percpu(void *__pdata); extern phys_addr_t per_cpu_ptr_to_phys(void *addr); extern void __bad_size_call_parameter(void); int arch_update_cpu_topology(void); static inline __attribute__((always_inline)) int numa_mem_id(void) { return numa_node_id(); } struct vm_area_struct; static inline __attribute__((always_inline)) int allocflags_to_migratetype(gfp_t gfp_flags) { ({ int __ret_warn_on = !!((gfp_flags & ((( gfp_t)0x80000u)|(( gfp_t)0x08u))) == ((( gfp_t)0x80000u)|(( gfp_t)0x08u))); if (__builtin_constant_p((((__builtin_constant_p(__ret_warn_on) ? !!(__ret_warn_on) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/gfp.h", .line = 152, }; ______r = __builtin_expect(!!(__ret_warn_on), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(__ret_warn_on) ? !!(__ret_warn_on) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/gfp.h", .line = 152, }; ______r = __builtin_expect(!!(__ret_warn_on), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/gfp.h", .line = 152, }; ______r = !!(((__builtin_constant_p(__ret_warn_on) ? !!(__ret_warn_on) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/gfp.h", .line = 152, }; ______r = __builtin_expect(!!(__ret_warn_on), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) warn_slowpath_null("include/linux/gfp.h", 152); (__builtin_constant_p(__ret_warn_on) ? !!(__ret_warn_on) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/gfp.h", .line = 152, }; ______r = __builtin_expect(!!(__ret_warn_on), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })); }); if (__builtin_constant_p((((__builtin_constant_p(page_group_by_mobility_disabled) ? !!(page_group_by_mobility_disabled) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/gfp.h", .line = 154, }; ______r = __builtin_expect(!!(page_group_by_mobility_disabled), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(page_group_by_mobility_disabled) ? !!(page_group_by_mobility_disabled) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/gfp.h", .line = 154, }; ______r = __builtin_expect(!!(page_group_by_mobility_disabled), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/gfp.h", .line = 154, }; ______r = !!(((__builtin_constant_p(page_group_by_mobility_disabled) ? !!(page_group_by_mobility_disabled) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/gfp.h", .line = 154, }; ______r = __builtin_expect(!!(page_group_by_mobility_disabled), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) return 0; return (((gfp_flags & (( gfp_t)0x08u)) != 0) << 1) | ((gfp_flags & (( gfp_t)0x80000u)) != 0); } static inline __attribute__((always_inline)) enum zone_type gfp_zone(gfp_t flags) { enum zone_type z; int bit = ( int) (flags & ((( gfp_t)0x01u)|(( gfp_t)0x02u)|(( gfp_t)0x04u)|(( gfp_t)0x08u))); z = (( (ZONE_NORMAL << 0 * 2) | (ZONE_DMA << 0x01u * 2) | (ZONE_HIGHMEM << 0x02u * 2) | (ZONE_NORMAL << 0x04u * 2) | (ZONE_NORMAL << 0x08u * 2) | (ZONE_DMA << (0x08u | 0x01u) * 2) | (ZONE_MOVABLE << (0x08u | 0x02u) * 2) | (ZONE_NORMAL << (0x08u | 0x04u) * 2) ) >> (bit * 2)) & ((1 << 2) - 1); do { (void)((( 1 << (0x01u | 0x02u) | 1 << (0x01u | 0x04u) | 1 << (0x04u | 0x02u) | 1 << (0x01u | 0x04u | 0x02u) | 1 << (0x08u | 0x02u | 0x01u) | 1 << (0x08u | 0x04u | 0x01u) | 1 << (0x08u | 0x04u | 0x02u) | 1 << (0x08u | 0x04u | 0x01u | 0x02u) ) >> bit) & 1); } while (0); return z; } static inline __attribute__((always_inline)) int gfp_zonelist(gfp_t flags) { if (__builtin_constant_p(((0 && (__builtin_constant_p(flags & (( gfp_t)0x40000u)) ? !!(flags & (( gfp_t)0x40000u)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/gfp.h", .line = 265, }; ______r = __builtin_expect(!!(flags & (( gfp_t)0x40000u)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!((0 && (__builtin_constant_p(flags & (( gfp_t)0x40000u)) ? !!(flags & (( gfp_t)0x40000u)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/gfp.h", .line = 265, }; ______r = __builtin_expect(!!(flags & (( gfp_t)0x40000u)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/gfp.h", .line = 265, }; ______r = !!((0 && (__builtin_constant_p(flags & (( gfp_t)0x40000u)) ? !!(flags & (( gfp_t)0x40000u)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/gfp.h", .line = 265, }; ______r = __builtin_expect(!!(flags & (( gfp_t)0x40000u)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) return 1; return 0; } static inline __attribute__((always_inline)) struct zonelist *node_zonelist(int nid, gfp_t flags) { return (&contig_page_data)->node_zonelists + gfp_zonelist(flags); } static inline __attribute__((always_inline)) void arch_free_page(struct page *page, int order) { } static inline __attribute__((always_inline)) void arch_alloc_page(struct page *page, int order) { } struct page * __alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order, struct zonelist *zonelist, nodemask_t *nodemask); static inline __attribute__((always_inline)) struct page * __alloc_pages(gfp_t gfp_mask, unsigned int order, struct zonelist *zonelist) { return __alloc_pages_nodemask(gfp_mask, order, zonelist, ((void *)0)); } static inline __attribute__((always_inline)) struct page *alloc_pages_node(int nid, gfp_t gfp_mask, unsigned int order) { if (__builtin_constant_p(((nid < 0))) ? !!((nid < 0)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/gfp.h", .line = 307, }; ______r = !!((nid < 0)); ______f.miss_hit[______r]++; ______r; })) nid = numa_node_id(); return __alloc_pages(gfp_mask, order, node_zonelist(nid, gfp_mask)); } static inline __attribute__((always_inline)) struct page *alloc_pages_exact_node(int nid, gfp_t gfp_mask, unsigned int order) { do { (void)(nid < 0 || nid >= (1 << 0)); } while (0); return __alloc_pages(gfp_mask, order, node_zonelist(nid, gfp_mask)); } extern unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order); extern unsigned long get_zeroed_page(gfp_t gfp_mask); void *alloc_pages_exact(size_t size, gfp_t gfp_mask); void free_pages_exact(void *virt, size_t size); void *alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask); extern void __free_pages(struct page *page, unsigned int order); extern void free_pages(unsigned long addr, unsigned int order); extern void free_hot_cold_page(struct page *page, int cold); void page_alloc_init(void); void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp); void drain_all_pages(void); void drain_local_pages(void *dummy); extern gfp_t gfp_allowed_mask; extern void pm_restrict_gfp_mask(void); extern void pm_restore_gfp_mask(void); enum irqreturn { IRQ_NONE = (0 << 0), IRQ_HANDLED = (1 << 0), IRQ_WAKE_THREAD = (1 << 1), }; typedef enum irqreturn irqreturn_t; extern int nr_irqs; extern struct irq_desc *irq_to_desc(unsigned int irq); unsigned int irq_get_next_irq(unsigned int offset); static inline __attribute__((always_inline)) int irq_canonicalize(int irq) { return ((irq == 2) ? 9 : irq); } extern void irq_ctx_init(int cpu); extern void fixup_irqs(void); extern void irq_force_complete_move(int); extern void (*x86_platform_ipi_callback)(void); extern void native_init_IRQ(void); extern bool handle_irq(unsigned irq, struct pt_regs *regs); extern unsigned int do_IRQ(struct pt_regs *regs); extern unsigned long used_vectors[(((256) + (8 * sizeof(long)) - 1) / (8 * sizeof(long)))]; extern int vector_used_by_percpu_irq(unsigned int vector); extern void init_ISA_irqs(void); extern __attribute__((section(".data..percpu" ""))) __typeof__(struct pt_regs *) irq_regs; static inline __attribute__((always_inline)) struct pt_regs *get_irq_regs(void) { return ({ typeof(irq_regs) pfo_ret__; switch (sizeof(irq_regs)) { case 1: asm("mov" "b ""%%""fs"":" "%P" "1"",%0" : "=q" (pfo_ret__) : "m" (irq_regs)); break; case 2: asm("mov" "w ""%%""fs"":" "%P" "1"",%0" : "=r" (pfo_ret__) : "m" (irq_regs)); break; case 4: asm("mov" "l ""%%""fs"":" "%P" "1"",%0" : "=r" (pfo_ret__) : "m" (irq_regs)); break; case 8: asm("mov" "q ""%%""fs"":" "%P" "1"",%0" : "=r" (pfo_ret__) : "m" (irq_regs)); break; default: __bad_percpu_size(); } pfo_ret__; }); } static inline __attribute__((always_inline)) struct pt_regs *set_irq_regs(struct pt_regs *new_regs) { struct pt_regs *old_regs; old_regs = get_irq_regs(); do { typedef typeof(irq_regs) pto_T__; if (__builtin_constant_p(((0))) ? !!((0)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/irq_regs.h", .line = 26, }; ______r = !!((0)); ______f.miss_hit[______r]++; ______r; })) { pto_T__ pto_tmp__; pto_tmp__ = (new_regs); (void)pto_tmp__; } switch (sizeof(irq_regs)) { case 1: asm("mov" "b %1,""%%""fs"":" "%P" "0" : "+m" (irq_regs) : "qi" ((pto_T__)(new_regs))); break; case 2: asm("mov" "w %1,""%%""fs"":" "%P" "0" : "+m" (irq_regs) : "ri" ((pto_T__)(new_regs))); break; case 4: asm("mov" "l %1,""%%""fs"":" "%P" "0" : "+m" (irq_regs) : "ri" ((pto_T__)(new_regs))); break; case 8: asm("mov" "q %1,""%%""fs"":" "%P" "0" : "+m" (irq_regs) : "re" ((pto_T__)(new_regs))); break; default: __bad_percpu_size(); } } while (0); return old_regs; } struct seq_file; struct irq_desc; struct irq_data; typedef void (*irq_flow_handler_t)(unsigned int irq, struct irq_desc *desc); typedef void (*irq_preflow_handler_t)(struct irq_data *data); enum { IRQ_TYPE_NONE = 0x00000000, IRQ_TYPE_EDGE_RISING = 0x00000001, IRQ_TYPE_EDGE_FALLING = 0x00000002, IRQ_TYPE_EDGE_BOTH = (IRQ_TYPE_EDGE_FALLING | IRQ_TYPE_EDGE_RISING), IRQ_TYPE_LEVEL_HIGH = 0x00000004, IRQ_TYPE_LEVEL_LOW = 0x00000008, IRQ_TYPE_LEVEL_MASK = (IRQ_TYPE_LEVEL_LOW | IRQ_TYPE_LEVEL_HIGH), IRQ_TYPE_SENSE_MASK = 0x0000000f, IRQ_TYPE_PROBE = 0x00000010, IRQ_LEVEL = (1 << 8), IRQ_PER_CPU = (1 << 9), IRQ_NOPROBE = (1 << 10), IRQ_NOREQUEST = (1 << 11), IRQ_NOAUTOEN = (1 << 12), IRQ_NO_BALANCING = (1 << 13), IRQ_MOVE_PCNTXT = (1 << 14), IRQ_NESTED_THREAD = (1 << 15), IRQ_NOTHREAD = (1 << 16), }; static inline __attribute__((always_inline)) __attribute__((deprecated)) bool CHECK_IRQ_PER_CPU(unsigned int status) { return status & IRQ_PER_CPU; } enum { IRQ_SET_MASK_OK = 0, IRQ_SET_MASK_OK_NOCOPY, }; struct msi_desc; struct irq_data { unsigned int irq; unsigned int node; unsigned int state_use_accessors; struct irq_chip *chip; void *handler_data; void *chip_data; struct msi_desc *msi_desc; cpumask_var_t affinity; }; enum { IRQD_TRIGGER_MASK = 0xf, IRQD_SETAFFINITY_PENDING = (1 << 8), IRQD_NO_BALANCING = (1 << 10), IRQD_PER_CPU = (1 << 11), IRQD_AFFINITY_SET = (1 << 12), IRQD_LEVEL = (1 << 13), IRQD_WAKEUP_STATE = (1 << 14), IRQD_MOVE_PCNTXT = (1 << 15), IRQD_IRQ_DISABLED = (1 << 16), IRQD_IRQ_MASKED = (1 << 17), IRQD_IRQ_INPROGRESS = (1 << 18), }; static inline __attribute__((always_inline)) bool irqd_is_setaffinity_pending(struct irq_data *d) { return d->state_use_accessors & IRQD_SETAFFINITY_PENDING; } static inline __attribute__((always_inline)) bool irqd_is_per_cpu(struct irq_data *d) { return d->state_use_accessors & IRQD_PER_CPU; } static inline __attribute__((always_inline)) bool irqd_can_balance(struct irq_data *d) { return !(d->state_use_accessors & (IRQD_PER_CPU | IRQD_NO_BALANCING)); } static inline __attribute__((always_inline)) bool irqd_affinity_was_set(struct irq_data *d) { return d->state_use_accessors & IRQD_AFFINITY_SET; } static inline __attribute__((always_inline)) void irqd_mark_affinity_was_set(struct irq_data *d) { d->state_use_accessors |= IRQD_AFFINITY_SET; } static inline __attribute__((always_inline)) u32 irqd_get_trigger_type(struct irq_data *d) { return d->state_use_accessors & IRQD_TRIGGER_MASK; } static inline __attribute__((always_inline)) void irqd_set_trigger_type(struct irq_data *d, u32 type) { d->state_use_accessors &= ~IRQD_TRIGGER_MASK; d->state_use_accessors |= type & IRQD_TRIGGER_MASK; } static inline __attribute__((always_inline)) bool irqd_is_level_type(struct irq_data *d) { return d->state_use_accessors & IRQD_LEVEL; } static inline __attribute__((always_inline)) bool irqd_is_wakeup_set(struct irq_data *d) { return d->state_use_accessors & IRQD_WAKEUP_STATE; } static inline __attribute__((always_inline)) bool irqd_can_move_in_process_context(struct irq_data *d) { return d->state_use_accessors & IRQD_MOVE_PCNTXT; } static inline __attribute__((always_inline)) bool irqd_irq_disabled(struct irq_data *d) { return d->state_use_accessors & IRQD_IRQ_DISABLED; } static inline __attribute__((always_inline)) bool irqd_irq_masked(struct irq_data *d) { return d->state_use_accessors & IRQD_IRQ_MASKED; } static inline __attribute__((always_inline)) bool irqd_irq_inprogress(struct irq_data *d) { return d->state_use_accessors & IRQD_IRQ_INPROGRESS; } static inline __attribute__((always_inline)) void irqd_set_chained_irq_inprogress(struct irq_data *d) { d->state_use_accessors |= IRQD_IRQ_INPROGRESS; } static inline __attribute__((always_inline)) void irqd_clr_chained_irq_inprogress(struct irq_data *d) { d->state_use_accessors &= ~IRQD_IRQ_INPROGRESS; } struct irq_chip { const char *name; unsigned int (*irq_startup)(struct irq_data *data); void (*irq_shutdown)(struct irq_data *data); void (*irq_enable)(struct irq_data *data); void (*irq_disable)(struct irq_data *data); void (*irq_ack)(struct irq_data *data); void (*irq_mask)(struct irq_data *data); void (*irq_mask_ack)(struct irq_data *data); void (*irq_unmask)(struct irq_data *data); void (*irq_eoi)(struct irq_data *data); int (*irq_set_affinity)(struct irq_data *data, const struct cpumask *dest, bool force); int (*irq_retrigger)(struct irq_data *data); int (*irq_set_type)(struct irq_data *data, unsigned int flow_type); int (*irq_set_wake)(struct irq_data *data, unsigned int on); void (*irq_bus_lock)(struct irq_data *data); void (*irq_bus_sync_unlock)(struct irq_data *data); void (*irq_cpu_online)(struct irq_data *data); void (*irq_cpu_offline)(struct irq_data *data); void (*irq_suspend)(struct irq_data *data); void (*irq_resume)(struct irq_data *data); void (*irq_pm_shutdown)(struct irq_data *data); void (*irq_print_chip)(struct irq_data *data, struct seq_file *p); unsigned long flags; }; enum { IRQCHIP_SET_TYPE_MASKED = (1 << 0), IRQCHIP_EOI_IF_HANDLED = (1 << 1), IRQCHIP_MASK_ON_SUSPEND = (1 << 2), IRQCHIP_ONOFFLINE_ENABLED = (1 << 3), }; struct irq_affinity_notify; struct proc_dir_entry; struct timer_rand_state; struct irq_desc { struct irq_data irq_data; struct timer_rand_state *timer_rand_state; unsigned int *kstat_irqs; irq_flow_handler_t handle_irq; struct irqaction *action; unsigned int status_use_accessors; unsigned int core_internal_state__do_not_mess_with_it; unsigned int depth; unsigned int wake_depth; unsigned int irq_count; unsigned long last_unhandled; unsigned int irqs_unhandled; raw_spinlock_t lock; const struct cpumask *affinity_hint; struct irq_affinity_notify *affinity_notify; cpumask_var_t pending_mask; unsigned long threads_oneshot; atomic_t threads_active; wait_queue_head_t wait_for_threads; struct proc_dir_entry *dir; const char *name; } __attribute__((__aligned__(1 << (6)))); extern struct irq_desc irq_desc[((32 * 8) < ( 32 * 64 ) ? (256 + (32 * 8)) : (256 + ( 32 * 64 )))]; static inline __attribute__((always_inline)) struct irq_data *irq_desc_get_irq_data(struct irq_desc *desc) { return &desc->irq_data; } static inline __attribute__((always_inline)) struct irq_chip *irq_desc_get_chip(struct irq_desc *desc) { return desc->irq_data.chip; } static inline __attribute__((always_inline)) void *irq_desc_get_chip_data(struct irq_desc *desc) { return desc->irq_data.chip_data; } static inline __attribute__((always_inline)) void *irq_desc_get_handler_data(struct irq_desc *desc) { return desc->irq_data.handler_data; } static inline __attribute__((always_inline)) struct msi_desc *irq_desc_get_msi_desc(struct irq_desc *desc) { return desc->irq_data.msi_desc; } static inline __attribute__((always_inline)) void generic_handle_irq_desc(unsigned int irq, struct irq_desc *desc) { desc->handle_irq(irq, desc); } int generic_handle_irq(unsigned int irq); static inline __attribute__((always_inline)) int irq_has_action(unsigned int irq) { struct irq_desc *desc = irq_to_desc(irq); return desc->action != ((void *)0); } static inline __attribute__((always_inline)) void __irq_set_handler_locked(unsigned int irq, irq_flow_handler_t handler) { struct irq_desc *desc; desc = irq_to_desc(irq); desc->handle_irq = handler; } static inline __attribute__((always_inline)) void __irq_set_chip_handler_name_locked(unsigned int irq, struct irq_chip *chip, irq_flow_handler_t handler, const char *name) { struct irq_desc *desc; desc = irq_to_desc(irq); irq_desc_get_irq_data(desc)->chip = chip; desc->handle_irq = handler; desc->name = name; } static inline __attribute__((always_inline)) int irq_balancing_disabled(unsigned int irq) { struct irq_desc *desc; desc = irq_to_desc(irq); return desc->status_use_accessors & (IRQ_PER_CPU | IRQ_NO_BALANCING); } static inline __attribute__((always_inline)) void irq_set_lockdep_class(unsigned int irq, struct lock_class_key *class) { struct irq_desc *desc = irq_to_desc(irq); if (__builtin_constant_p(((desc))) ? !!((desc)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/irqdesc.h", .line = 159, }; ______r = !!((desc)); ______f.miss_hit[______r]++; ______r; })) lockdep_init_map(&(&desc->lock)->dep_map, "class", class, 0); } struct proc_dir_entry; struct pt_regs; struct notifier_block; void create_prof_cpu_mask(struct proc_dir_entry *de); int create_proc_profile(void); enum profile_type { PROFILE_TASK_EXIT, PROFILE_MUNMAP }; extern int prof_on __attribute__((__section__(".data..read_mostly"))); int profile_init(void); int profile_setup(char *str); void profile_tick(int type); void profile_hits(int type, void *ip, unsigned int nr_hits); static inline __attribute__((always_inline)) void profile_hit(int type, void *ip) { if (__builtin_constant_p((((__builtin_constant_p(prof_on == type) ? !!(prof_on == type) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/profile.h", .line = 61, }; ______r = __builtin_expect(!!(prof_on == type), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(prof_on == type) ? !!(prof_on == type) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/profile.h", .line = 61, }; ______r = __builtin_expect(!!(prof_on == type), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/profile.h", .line = 61, }; ______r = !!(((__builtin_constant_p(prof_on == type) ? !!(prof_on == type) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/profile.h", .line = 61, }; ______r = __builtin_expect(!!(prof_on == type), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) profile_hits(type, ip, 1); } struct task_struct; struct mm_struct; void profile_task_exit(struct task_struct * task); int profile_handoff_task(struct task_struct * task); void profile_munmap(unsigned long addr); int task_handoff_register(struct notifier_block * n); int task_handoff_unregister(struct notifier_block * n); int profile_event_register(enum profile_type, struct notifier_block * n); int profile_event_unregister(enum profile_type, struct notifier_block * n); int register_timer_hook(int (*hook)(struct pt_regs *)); void unregister_timer_hook(int (*hook)(struct pt_regs *)); struct pt_regs; extern char _text[], _stext[], _etext[]; extern char _data[], _sdata[], _edata[]; extern char __bss_start[], __bss_stop[]; extern char __init_begin[], __init_end[]; extern char _sinittext[], _einittext[]; extern char _end[]; extern char __per_cpu_load[], __per_cpu_start[], __per_cpu_end[]; extern char __kprobes_text_start[], __kprobes_text_end[]; extern char __entry_text_start[], __entry_text_end[]; extern char __initdata_begin[], __initdata_end[]; extern char __start_rodata[], __end_rodata[]; extern char __ctors_start[], __ctors_end[]; static inline __attribute__((always_inline)) int arch_is_kernel_text(unsigned long addr) { return 0; } static inline __attribute__((always_inline)) int arch_is_kernel_data(unsigned long addr) { return 0; } struct exception_table_entry { unsigned long insn, fixup; }; extern int fixup_exception(struct pt_regs *regs); extern int __get_user_1(void); extern int __get_user_2(void); extern int __get_user_4(void); extern int __get_user_8(void); extern int __get_user_bad(void); extern void __put_user_bad(void); extern void __put_user_1(void); extern void __put_user_2(void); extern void __put_user_4(void); extern void __put_user_8(void); struct __large_struct { unsigned long buf[100]; }; extern struct movsl_mask { int mask; } __attribute__((__aligned__((1 << (6))))) movsl_mask; unsigned long __attribute__((warn_unused_result)) __copy_to_user_ll (void *to, const void *from, unsigned long n); unsigned long __attribute__((warn_unused_result)) __copy_from_user_ll (void *to, const void *from, unsigned long n); unsigned long __attribute__((warn_unused_result)) __copy_from_user_ll_nozero (void *to, const void *from, unsigned long n); unsigned long __attribute__((warn_unused_result)) __copy_from_user_ll_nocache (void *to, const void *from, unsigned long n); unsigned long __attribute__((warn_unused_result)) __copy_from_user_ll_nocache_nozero (void *to, const void *from, unsigned long n); static inline __attribute__((always_inline)) __attribute__((always_inline)) unsigned long __attribute__((warn_unused_result)) __copy_to_user_inatomic(void *to, const void *from, unsigned long n) { if (__builtin_constant_p(((__builtin_constant_p(n)))) ? !!((__builtin_constant_p(n))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/uaccess_32.h", .line = 46, }; ______r = !!((__builtin_constant_p(n))); ______f.miss_hit[______r]++; ______r; })) { unsigned long ret; switch (n) { case 1: do { ret = 0; (void)0; switch (1) { case 1: asm volatile("1: mov""b"" %""b""1,%2\n" "2:\n" ".section .fixup,\"ax\"\n" "3: mov %3,%0\n" " jmp 2b\n" ".previous\n" " .section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "1b" "," "3b" "\n" " .previous\n" : "=r"(ret) : "iq"(*(u8 *)from), "m" ((*(struct __large_struct *)((u8 *)to))), "i" (1), "0" (ret)); break; case 2: asm volatile("1: mov""w"" %""w""1,%2\n" "2:\n" ".section .fixup,\"ax\"\n" "3: mov %3,%0\n" " jmp 2b\n" ".previous\n" " .section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "1b" "," "3b" "\n" " .previous\n" : "=r"(ret) : "ir"(*(u8 *)from), "m" ((*(struct __large_struct *)((u8 *)to))), "i" (1), "0" (ret)); break; case 4: asm volatile("1: mov""l"" %""k""1,%2\n" "2:\n" ".section .fixup,\"ax\"\n" "3: mov %3,%0\n" " jmp 2b\n" ".previous\n" " .section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "1b" "," "3b" "\n" " .previous\n" : "=r"(ret) : "ir"(*(u8 *)from), "m" ((*(struct __large_struct *)((u8 *)to))), "i" (1), "0" (ret)); break; case 8: asm volatile("1: movl %%eax,0(%2)\n" "2: movl %%edx,4(%2)\n" "3:\n" ".section .fixup,\"ax\"\n" "4: movl %3,%0\n" " jmp 3b\n" ".previous\n" " .section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "1b" "," "4b" "\n" " .previous\n" " .section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "2b" "," "4b" "\n" " .previous\n" : "=r" (ret) : "A" ((__typeof__(*(u8 *)to))(*(u8 *)from)), "r" ((u8 *)to), "i" (1), "0" (ret)); break; default: __put_user_bad(); } } while (0) ; return ret; case 2: do { ret = 0; (void)0; switch (2) { case 1: asm volatile("1: mov""b"" %""b""1,%2\n" "2:\n" ".section .fixup,\"ax\"\n" "3: mov %3,%0\n" " jmp 2b\n" ".previous\n" " .section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "1b" "," "3b" "\n" " .previous\n" : "=r"(ret) : "iq"(*(u16 *)from), "m" ((*(struct __large_struct *)((u16 *)to))), "i" (2), "0" (ret)); break; case 2: asm volatile("1: mov""w"" %""w""1,%2\n" "2:\n" ".section .fixup,\"ax\"\n" "3: mov %3,%0\n" " jmp 2b\n" ".previous\n" " .section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "1b" "," "3b" "\n" " .previous\n" : "=r"(ret) : "ir"(*(u16 *)from), "m" ((*(struct __large_struct *)((u16 *)to))), "i" (2), "0" (ret)); break; case 4: asm volatile("1: mov""l"" %""k""1,%2\n" "2:\n" ".section .fixup,\"ax\"\n" "3: mov %3,%0\n" " jmp 2b\n" ".previous\n" " .section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "1b" "," "3b" "\n" " .previous\n" : "=r"(ret) : "ir"(*(u16 *)from), "m" ((*(struct __large_struct *)((u16 *)to))), "i" (2), "0" (ret)); break; case 8: asm volatile("1: movl %%eax,0(%2)\n" "2: movl %%edx,4(%2)\n" "3:\n" ".section .fixup,\"ax\"\n" "4: movl %3,%0\n" " jmp 3b\n" ".previous\n" " .section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "1b" "," "4b" "\n" " .previous\n" " .section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "2b" "," "4b" "\n" " .previous\n" : "=r" (ret) : "A" ((__typeof__(*(u16 *)to))(*(u16 *)from)), "r" ((u16 *)to), "i" (2), "0" (ret)); break; default: __put_user_bad(); } } while (0) ; return ret; case 4: do { ret = 0; (void)0; switch (4) { case 1: asm volatile("1: mov""b"" %""b""1,%2\n" "2:\n" ".section .fixup,\"ax\"\n" "3: mov %3,%0\n" " jmp 2b\n" ".previous\n" " .section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "1b" "," "3b" "\n" " .previous\n" : "=r"(ret) : "iq"(*(u32 *)from), "m" ((*(struct __large_struct *)((u32 *)to))), "i" (4), "0" (ret)); break; case 2: asm volatile("1: mov""w"" %""w""1,%2\n" "2:\n" ".section .fixup,\"ax\"\n" "3: mov %3,%0\n" " jmp 2b\n" ".previous\n" " .section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "1b" "," "3b" "\n" " .previous\n" : "=r"(ret) : "ir"(*(u32 *)from), "m" ((*(struct __large_struct *)((u32 *)to))), "i" (4), "0" (ret)); break; case 4: asm volatile("1: mov""l"" %""k""1,%2\n" "2:\n" ".section .fixup,\"ax\"\n" "3: mov %3,%0\n" " jmp 2b\n" ".previous\n" " .section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "1b" "," "3b" "\n" " .previous\n" : "=r"(ret) : "ir"(*(u32 *)from), "m" ((*(struct __large_struct *)((u32 *)to))), "i" (4), "0" (ret)); break; case 8: asm volatile("1: movl %%eax,0(%2)\n" "2: movl %%edx,4(%2)\n" "3:\n" ".section .fixup,\"ax\"\n" "4: movl %3,%0\n" " jmp 3b\n" ".previous\n" " .section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "1b" "," "4b" "\n" " .previous\n" " .section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "2b" "," "4b" "\n" " .previous\n" : "=r" (ret) : "A" ((__typeof__(*(u32 *)to))(*(u32 *)from)), "r" ((u32 *)to), "i" (4), "0" (ret)); break; default: __put_user_bad(); } } while (0) ; return ret; } } return __copy_to_user_ll(to, from, n); } static inline __attribute__((always_inline)) __attribute__((always_inline)) unsigned long __attribute__((warn_unused_result)) __copy_to_user(void *to, const void *from, unsigned long n) { might_fault(); return __copy_to_user_inatomic(to, from, n); } static inline __attribute__((always_inline)) __attribute__((always_inline)) unsigned long __copy_from_user_inatomic(void *to, const void *from, unsigned long n) { if (__builtin_constant_p(((__builtin_constant_p(n)))) ? !!((__builtin_constant_p(n))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/uaccess_32.h", .line = 96, }; ______r = !!((__builtin_constant_p(n))); ______f.miss_hit[______r]++; ______r; })) { unsigned long ret; switch (n) { case 1: do { ret = 0; (void)0; switch (1) { case 1: asm volatile("1: mov""b"" %2,%""b""1\n" "2:\n" ".section .fixup,\"ax\"\n" "3: mov %3,%0\n" " xor""b"" %""b""1,%""b""1\n" " jmp 2b\n" ".previous\n" " .section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "1b" "," "3b" "\n" " .previous\n" : "=r" (ret), "=q"(*(u8 *)to) : "m" ((*(struct __large_struct *)(from))), "i" (1), "0" (ret)); break; case 2: asm volatile("1: mov""w"" %2,%""w""1\n" "2:\n" ".section .fixup,\"ax\"\n" "3: mov %3,%0\n" " xor""w"" %""w""1,%""w""1\n" " jmp 2b\n" ".previous\n" " .section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "1b" "," "3b" "\n" " .previous\n" : "=r" (ret), "=r"(*(u8 *)to) : "m" ((*(struct __large_struct *)(from))), "i" (1), "0" (ret)); break; case 4: asm volatile("1: mov""l"" %2,%""k""1\n" "2:\n" ".section .fixup,\"ax\"\n" "3: mov %3,%0\n" " xor""l"" %""k""1,%""k""1\n" " jmp 2b\n" ".previous\n" " .section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "1b" "," "3b" "\n" " .previous\n" : "=r" (ret), "=r"(*(u8 *)to) : "m" ((*(struct __large_struct *)(from))), "i" (1), "0" (ret)); break; case 8: (*(u8 *)to) = __get_user_bad(); break; default: (*(u8 *)to) = __get_user_bad(); } } while (0); return ret; case 2: do { ret = 0; (void)0; switch (2) { case 1: asm volatile("1: mov""b"" %2,%""b""1\n" "2:\n" ".section .fixup,\"ax\"\n" "3: mov %3,%0\n" " xor""b"" %""b""1,%""b""1\n" " jmp 2b\n" ".previous\n" " .section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "1b" "," "3b" "\n" " .previous\n" : "=r" (ret), "=q"(*(u16 *)to) : "m" ((*(struct __large_struct *)(from))), "i" (2), "0" (ret)); break; case 2: asm volatile("1: mov""w"" %2,%""w""1\n" "2:\n" ".section .fixup,\"ax\"\n" "3: mov %3,%0\n" " xor""w"" %""w""1,%""w""1\n" " jmp 2b\n" ".previous\n" " .section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "1b" "," "3b" "\n" " .previous\n" : "=r" (ret), "=r"(*(u16 *)to) : "m" ((*(struct __large_struct *)(from))), "i" (2), "0" (ret)); break; case 4: asm volatile("1: mov""l"" %2,%""k""1\n" "2:\n" ".section .fixup,\"ax\"\n" "3: mov %3,%0\n" " xor""l"" %""k""1,%""k""1\n" " jmp 2b\n" ".previous\n" " .section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "1b" "," "3b" "\n" " .previous\n" : "=r" (ret), "=r"(*(u16 *)to) : "m" ((*(struct __large_struct *)(from))), "i" (2), "0" (ret)); break; case 8: (*(u16 *)to) = __get_user_bad(); break; default: (*(u16 *)to) = __get_user_bad(); } } while (0); return ret; case 4: do { ret = 0; (void)0; switch (4) { case 1: asm volatile("1: mov""b"" %2,%""b""1\n" "2:\n" ".section .fixup,\"ax\"\n" "3: mov %3,%0\n" " xor""b"" %""b""1,%""b""1\n" " jmp 2b\n" ".previous\n" " .section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "1b" "," "3b" "\n" " .previous\n" : "=r" (ret), "=q"(*(u32 *)to) : "m" ((*(struct __large_struct *)(from))), "i" (4), "0" (ret)); break; case 2: asm volatile("1: mov""w"" %2,%""w""1\n" "2:\n" ".section .fixup,\"ax\"\n" "3: mov %3,%0\n" " xor""w"" %""w""1,%""w""1\n" " jmp 2b\n" ".previous\n" " .section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "1b" "," "3b" "\n" " .previous\n" : "=r" (ret), "=r"(*(u32 *)to) : "m" ((*(struct __large_struct *)(from))), "i" (4), "0" (ret)); break; case 4: asm volatile("1: mov""l"" %2,%""k""1\n" "2:\n" ".section .fixup,\"ax\"\n" "3: mov %3,%0\n" " xor""l"" %""k""1,%""k""1\n" " jmp 2b\n" ".previous\n" " .section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "1b" "," "3b" "\n" " .previous\n" : "=r" (ret), "=r"(*(u32 *)to) : "m" ((*(struct __large_struct *)(from))), "i" (4), "0" (ret)); break; case 8: (*(u32 *)to) = __get_user_bad(); break; default: (*(u32 *)to) = __get_user_bad(); } } while (0); return ret; } } return __copy_from_user_ll_nozero(to, from, n); } static inline __attribute__((always_inline)) __attribute__((always_inline)) unsigned long __copy_from_user(void *to, const void *from, unsigned long n) { might_fault(); if (__builtin_constant_p(((__builtin_constant_p(n)))) ? !!((__builtin_constant_p(n))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/uaccess_32.h", .line = 140, }; ______r = !!((__builtin_constant_p(n))); ______f.miss_hit[______r]++; ______r; })) { unsigned long ret; switch (n) { case 1: do { ret = 0; (void)0; switch (1) { case 1: asm volatile("1: mov""b"" %2,%""b""1\n" "2:\n" ".section .fixup,\"ax\"\n" "3: mov %3,%0\n" " xor""b"" %""b""1,%""b""1\n" " jmp 2b\n" ".previous\n" " .section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "1b" "," "3b" "\n" " .previous\n" : "=r" (ret), "=q"(*(u8 *)to) : "m" ((*(struct __large_struct *)(from))), "i" (1), "0" (ret)); break; case 2: asm volatile("1: mov""w"" %2,%""w""1\n" "2:\n" ".section .fixup,\"ax\"\n" "3: mov %3,%0\n" " xor""w"" %""w""1,%""w""1\n" " jmp 2b\n" ".previous\n" " .section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "1b" "," "3b" "\n" " .previous\n" : "=r" (ret), "=r"(*(u8 *)to) : "m" ((*(struct __large_struct *)(from))), "i" (1), "0" (ret)); break; case 4: asm volatile("1: mov""l"" %2,%""k""1\n" "2:\n" ".section .fixup,\"ax\"\n" "3: mov %3,%0\n" " xor""l"" %""k""1,%""k""1\n" " jmp 2b\n" ".previous\n" " .section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "1b" "," "3b" "\n" " .previous\n" : "=r" (ret), "=r"(*(u8 *)to) : "m" ((*(struct __large_struct *)(from))), "i" (1), "0" (ret)); break; case 8: (*(u8 *)to) = __get_user_bad(); break; default: (*(u8 *)to) = __get_user_bad(); } } while (0); return ret; case 2: do { ret = 0; (void)0; switch (2) { case 1: asm volatile("1: mov""b"" %2,%""b""1\n" "2:\n" ".section .fixup,\"ax\"\n" "3: mov %3,%0\n" " xor""b"" %""b""1,%""b""1\n" " jmp 2b\n" ".previous\n" " .section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "1b" "," "3b" "\n" " .previous\n" : "=r" (ret), "=q"(*(u16 *)to) : "m" ((*(struct __large_struct *)(from))), "i" (2), "0" (ret)); break; case 2: asm volatile("1: mov""w"" %2,%""w""1\n" "2:\n" ".section .fixup,\"ax\"\n" "3: mov %3,%0\n" " xor""w"" %""w""1,%""w""1\n" " jmp 2b\n" ".previous\n" " .section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "1b" "," "3b" "\n" " .previous\n" : "=r" (ret), "=r"(*(u16 *)to) : "m" ((*(struct __large_struct *)(from))), "i" (2), "0" (ret)); break; case 4: asm volatile("1: mov""l"" %2,%""k""1\n" "2:\n" ".section .fixup,\"ax\"\n" "3: mov %3,%0\n" " xor""l"" %""k""1,%""k""1\n" " jmp 2b\n" ".previous\n" " .section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "1b" "," "3b" "\n" " .previous\n" : "=r" (ret), "=r"(*(u16 *)to) : "m" ((*(struct __large_struct *)(from))), "i" (2), "0" (ret)); break; case 8: (*(u16 *)to) = __get_user_bad(); break; default: (*(u16 *)to) = __get_user_bad(); } } while (0); return ret; case 4: do { ret = 0; (void)0; switch (4) { case 1: asm volatile("1: mov""b"" %2,%""b""1\n" "2:\n" ".section .fixup,\"ax\"\n" "3: mov %3,%0\n" " xor""b"" %""b""1,%""b""1\n" " jmp 2b\n" ".previous\n" " .section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "1b" "," "3b" "\n" " .previous\n" : "=r" (ret), "=q"(*(u32 *)to) : "m" ((*(struct __large_struct *)(from))), "i" (4), "0" (ret)); break; case 2: asm volatile("1: mov""w"" %2,%""w""1\n" "2:\n" ".section .fixup,\"ax\"\n" "3: mov %3,%0\n" " xor""w"" %""w""1,%""w""1\n" " jmp 2b\n" ".previous\n" " .section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "1b" "," "3b" "\n" " .previous\n" : "=r" (ret), "=r"(*(u32 *)to) : "m" ((*(struct __large_struct *)(from))), "i" (4), "0" (ret)); break; case 4: asm volatile("1: mov""l"" %2,%""k""1\n" "2:\n" ".section .fixup,\"ax\"\n" "3: mov %3,%0\n" " xor""l"" %""k""1,%""k""1\n" " jmp 2b\n" ".previous\n" " .section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "1b" "," "3b" "\n" " .previous\n" : "=r" (ret), "=r"(*(u32 *)to) : "m" ((*(struct __large_struct *)(from))), "i" (4), "0" (ret)); break; case 8: (*(u32 *)to) = __get_user_bad(); break; default: (*(u32 *)to) = __get_user_bad(); } } while (0); return ret; } } return __copy_from_user_ll(to, from, n); } static inline __attribute__((always_inline)) __attribute__((always_inline)) unsigned long __copy_from_user_nocache(void *to, const void *from, unsigned long n) { might_fault(); if (__builtin_constant_p(((__builtin_constant_p(n)))) ? !!((__builtin_constant_p(n))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/uaccess_32.h", .line = 162, }; ______r = !!((__builtin_constant_p(n))); ______f.miss_hit[______r]++; ______r; })) { unsigned long ret; switch (n) { case 1: do { ret = 0; (void)0; switch (1) { case 1: asm volatile("1: mov""b"" %2,%""b""1\n" "2:\n" ".section .fixup,\"ax\"\n" "3: mov %3,%0\n" " xor""b"" %""b""1,%""b""1\n" " jmp 2b\n" ".previous\n" " .section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "1b" "," "3b" "\n" " .previous\n" : "=r" (ret), "=q"(*(u8 *)to) : "m" ((*(struct __large_struct *)(from))), "i" (1), "0" (ret)); break; case 2: asm volatile("1: mov""w"" %2,%""w""1\n" "2:\n" ".section .fixup,\"ax\"\n" "3: mov %3,%0\n" " xor""w"" %""w""1,%""w""1\n" " jmp 2b\n" ".previous\n" " .section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "1b" "," "3b" "\n" " .previous\n" : "=r" (ret), "=r"(*(u8 *)to) : "m" ((*(struct __large_struct *)(from))), "i" (1), "0" (ret)); break; case 4: asm volatile("1: mov""l"" %2,%""k""1\n" "2:\n" ".section .fixup,\"ax\"\n" "3: mov %3,%0\n" " xor""l"" %""k""1,%""k""1\n" " jmp 2b\n" ".previous\n" " .section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "1b" "," "3b" "\n" " .previous\n" : "=r" (ret), "=r"(*(u8 *)to) : "m" ((*(struct __large_struct *)(from))), "i" (1), "0" (ret)); break; case 8: (*(u8 *)to) = __get_user_bad(); break; default: (*(u8 *)to) = __get_user_bad(); } } while (0); return ret; case 2: do { ret = 0; (void)0; switch (2) { case 1: asm volatile("1: mov""b"" %2,%""b""1\n" "2:\n" ".section .fixup,\"ax\"\n" "3: mov %3,%0\n" " xor""b"" %""b""1,%""b""1\n" " jmp 2b\n" ".previous\n" " .section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "1b" "," "3b" "\n" " .previous\n" : "=r" (ret), "=q"(*(u16 *)to) : "m" ((*(struct __large_struct *)(from))), "i" (2), "0" (ret)); break; case 2: asm volatile("1: mov""w"" %2,%""w""1\n" "2:\n" ".section .fixup,\"ax\"\n" "3: mov %3,%0\n" " xor""w"" %""w""1,%""w""1\n" " jmp 2b\n" ".previous\n" " .section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "1b" "," "3b" "\n" " .previous\n" : "=r" (ret), "=r"(*(u16 *)to) : "m" ((*(struct __large_struct *)(from))), "i" (2), "0" (ret)); break; case 4: asm volatile("1: mov""l"" %2,%""k""1\n" "2:\n" ".section .fixup,\"ax\"\n" "3: mov %3,%0\n" " xor""l"" %""k""1,%""k""1\n" " jmp 2b\n" ".previous\n" " .section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "1b" "," "3b" "\n" " .previous\n" : "=r" (ret), "=r"(*(u16 *)to) : "m" ((*(struct __large_struct *)(from))), "i" (2), "0" (ret)); break; case 8: (*(u16 *)to) = __get_user_bad(); break; default: (*(u16 *)to) = __get_user_bad(); } } while (0); return ret; case 4: do { ret = 0; (void)0; switch (4) { case 1: asm volatile("1: mov""b"" %2,%""b""1\n" "2:\n" ".section .fixup,\"ax\"\n" "3: mov %3,%0\n" " xor""b"" %""b""1,%""b""1\n" " jmp 2b\n" ".previous\n" " .section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "1b" "," "3b" "\n" " .previous\n" : "=r" (ret), "=q"(*(u32 *)to) : "m" ((*(struct __large_struct *)(from))), "i" (4), "0" (ret)); break; case 2: asm volatile("1: mov""w"" %2,%""w""1\n" "2:\n" ".section .fixup,\"ax\"\n" "3: mov %3,%0\n" " xor""w"" %""w""1,%""w""1\n" " jmp 2b\n" ".previous\n" " .section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "1b" "," "3b" "\n" " .previous\n" : "=r" (ret), "=r"(*(u32 *)to) : "m" ((*(struct __large_struct *)(from))), "i" (4), "0" (ret)); break; case 4: asm volatile("1: mov""l"" %2,%""k""1\n" "2:\n" ".section .fixup,\"ax\"\n" "3: mov %3,%0\n" " xor""l"" %""k""1,%""k""1\n" " jmp 2b\n" ".previous\n" " .section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "1b" "," "3b" "\n" " .previous\n" : "=r" (ret), "=r"(*(u32 *)to) : "m" ((*(struct __large_struct *)(from))), "i" (4), "0" (ret)); break; case 8: (*(u32 *)to) = __get_user_bad(); break; default: (*(u32 *)to) = __get_user_bad(); } } while (0); return ret; } } return __copy_from_user_ll_nocache(to, from, n); } static inline __attribute__((always_inline)) __attribute__((always_inline)) unsigned long __copy_from_user_inatomic_nocache(void *to, const void *from, unsigned long n) { return __copy_from_user_ll_nocache_nozero(to, from, n); } unsigned long __attribute__((warn_unused_result)) copy_to_user(void *to, const void *from, unsigned long n); unsigned long __attribute__((warn_unused_result)) _copy_from_user(void *to, const void *from, unsigned long n); extern void copy_from_user_overflow(void) __attribute__((warning("copy_from_user() buffer size is not provably correct"))) ; static inline __attribute__((always_inline)) unsigned long __attribute__((warn_unused_result)) copy_from_user(void *to, const void *from, unsigned long n) { int sz = __builtin_object_size(to, 0); if (__builtin_constant_p((((__builtin_constant_p(sz == -1 || sz >= n) ? !!(sz == -1 || sz >= n) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/uaccess_32.h", .line = 208, }; ______r = __builtin_expect(!!(sz == -1 || sz >= n), 1); ftrace_likely_update(&______f, ______r, 1); ______r; }))))) ? !!(((__builtin_constant_p(sz == -1 || sz >= n) ? !!(sz == -1 || sz >= n) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/uaccess_32.h", .line = 208, }; ______r = __builtin_expect(!!(sz == -1 || sz >= n), 1); ftrace_likely_update(&______f, ______r, 1); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/uaccess_32.h", .line = 208, }; ______r = !!(((__builtin_constant_p(sz == -1 || sz >= n) ? !!(sz == -1 || sz >= n) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/uaccess_32.h", .line = 208, }; ______r = __builtin_expect(!!(sz == -1 || sz >= n), 1); ftrace_likely_update(&______f, ______r, 1); ______r; })))); ______f.miss_hit[______r]++; ______r; })) n = _copy_from_user(to, from, n); else copy_from_user_overflow(); return n; } long __attribute__((warn_unused_result)) strncpy_from_user(char *dst, const char *src, long count); long __attribute__((warn_unused_result)) __strncpy_from_user(char *dst, const char *src, long count); long strnlen_user(const char *str, long n); unsigned long __attribute__((warn_unused_result)) clear_user(void *mem, unsigned long len); unsigned long __attribute__((warn_unused_result)) __clear_user(void *mem, unsigned long len); extern char __brk_base[], __brk_limit[]; extern struct exception_table_entry __stop___ex_table[]; extern void apic_timer_interrupt(void); extern void x86_platform_ipi(void); extern void error_interrupt(void); extern void irq_work_interrupt(void); extern void spurious_interrupt(void); extern void thermal_interrupt(void); extern void reschedule_interrupt(void); extern void mce_self_interrupt(void); extern void invalidate_interrupt(void); extern void invalidate_interrupt0(void); extern void invalidate_interrupt1(void); extern void invalidate_interrupt2(void); extern void invalidate_interrupt3(void); extern void invalidate_interrupt4(void); extern void invalidate_interrupt5(void); extern void invalidate_interrupt6(void); extern void invalidate_interrupt7(void); extern void invalidate_interrupt8(void); extern void invalidate_interrupt9(void); extern void invalidate_interrupt10(void); extern void invalidate_interrupt11(void); extern void invalidate_interrupt12(void); extern void invalidate_interrupt13(void); extern void invalidate_interrupt14(void); extern void invalidate_interrupt15(void); extern void invalidate_interrupt16(void); extern void invalidate_interrupt17(void); extern void invalidate_interrupt18(void); extern void invalidate_interrupt19(void); extern void invalidate_interrupt20(void); extern void invalidate_interrupt21(void); extern void invalidate_interrupt22(void); extern void invalidate_interrupt23(void); extern void invalidate_interrupt24(void); extern void invalidate_interrupt25(void); extern void invalidate_interrupt26(void); extern void invalidate_interrupt27(void); extern void invalidate_interrupt28(void); extern void invalidate_interrupt29(void); extern void invalidate_interrupt30(void); extern void invalidate_interrupt31(void); extern void irq_move_cleanup_interrupt(void); extern void reboot_interrupt(void); extern void threshold_interrupt(void); extern void call_function_interrupt(void); extern void call_function_single_interrupt(void); extern unsigned long io_apic_irqs; extern void init_VISWS_APIC_irqs(void); extern void setup_IO_APIC(void); extern void disable_IO_APIC(void); struct io_apic_irq_attr { int ioapic; int ioapic_pin; int trigger; int polarity; }; static inline __attribute__((always_inline)) void set_io_apic_irq_attr(struct io_apic_irq_attr *irq_attr, int ioapic, int ioapic_pin, int trigger, int polarity) { irq_attr->ioapic = ioapic; irq_attr->ioapic_pin = ioapic_pin; irq_attr->trigger = trigger; irq_attr->polarity = polarity; } struct irq_2_iommu { struct intel_iommu *iommu; u16 irte_index; u16 sub_handle; u8 irte_mask; }; struct irq_cfg { struct irq_pin_list *irq_2_pin; cpumask_var_t domain; cpumask_var_t old_domain; u8 vector; u8 move_in_progress : 1; }; extern int assign_irq_vector(int, struct irq_cfg *, const struct cpumask *); extern void send_cleanup_vector(struct irq_cfg *); struct irq_data; int __ioapic_set_affinity(struct irq_data *, const struct cpumask *, unsigned int *dest_id); extern int IO_APIC_get_PCI_irq_vector(int bus, int devfn, int pin, struct io_apic_irq_attr *irq_attr); extern void setup_ioapic_dest(void); extern void enable_IO_APIC(void); extern atomic_t irq_err_count; extern atomic_t irq_mis_count; extern void eisa_set_level_irq(unsigned int irq); extern void smp_apic_timer_interrupt(struct pt_regs *); extern void smp_spurious_interrupt(struct pt_regs *); extern void smp_x86_platform_ipi(struct pt_regs *); extern void smp_error_interrupt(struct pt_regs *); extern __attribute__((regparm(0))) void smp_irq_move_cleanup_interrupt(void); extern void smp_reschedule_interrupt(struct pt_regs *); extern void smp_call_function_interrupt(struct pt_regs *); extern void smp_call_function_single_interrupt(struct pt_regs *); extern void smp_invalidate_interrupt(struct pt_regs *); extern void (*__attribute__ ((__section__(".init.rodata"))) interrupt[256 -0x20])(void); typedef int vector_irq_t[256]; extern __attribute__((section(".data..percpu" ""))) __typeof__(vector_irq_t) vector_irq; extern void setup_vector_irq(int cpu); extern void lock_vector_lock(void); extern void unlock_vector_lock(void); extern void __setup_vector_irq(int cpu); struct irqaction; extern int setup_irq(unsigned int irq, struct irqaction *new); extern void remove_irq(unsigned int irq, struct irqaction *act); extern void irq_cpu_online(void); extern void irq_cpu_offline(void); extern int __irq_set_affinity_locked(struct irq_data *data, const struct cpumask *cpumask); void irq_move_irq(struct irq_data *data); void irq_move_masked_irq(struct irq_data *data); extern int no_irq_affinity; extern void handle_level_irq(unsigned int irq, struct irq_desc *desc); extern void handle_fasteoi_irq(unsigned int irq, struct irq_desc *desc); extern void handle_edge_irq(unsigned int irq, struct irq_desc *desc); extern void handle_edge_eoi_irq(unsigned int irq, struct irq_desc *desc); extern void handle_simple_irq(unsigned int irq, struct irq_desc *desc); extern void handle_percpu_irq(unsigned int irq, struct irq_desc *desc); extern void handle_bad_irq(unsigned int irq, struct irq_desc *desc); extern void handle_nested_irq(unsigned int irq); extern void note_interrupt(unsigned int irq, struct irq_desc *desc, irqreturn_t action_ret); extern int noirqdebug_setup(char *str); extern int can_request_irq(unsigned int irq, unsigned long irqflags); extern struct irq_chip no_irq_chip; extern struct irq_chip dummy_irq_chip; extern void irq_set_chip_and_handler_name(unsigned int irq, struct irq_chip *chip, irq_flow_handler_t handle, const char *name); static inline __attribute__((always_inline)) void irq_set_chip_and_handler(unsigned int irq, struct irq_chip *chip, irq_flow_handler_t handle) { irq_set_chip_and_handler_name(irq, chip, handle, ((void *)0)); } extern void __irq_set_handler(unsigned int irq, irq_flow_handler_t handle, int is_chained, const char *name); static inline __attribute__((always_inline)) void irq_set_handler(unsigned int irq, irq_flow_handler_t handle) { __irq_set_handler(irq, handle, 0, ((void *)0)); } static inline __attribute__((always_inline)) void irq_set_chained_handler(unsigned int irq, irq_flow_handler_t handle) { __irq_set_handler(irq, handle, 1, ((void *)0)); } void irq_modify_status(unsigned int irq, unsigned long clr, unsigned long set); static inline __attribute__((always_inline)) void irq_set_status_flags(unsigned int irq, unsigned long set) { irq_modify_status(irq, 0, set); } static inline __attribute__((always_inline)) void irq_clear_status_flags(unsigned int irq, unsigned long clr) { irq_modify_status(irq, clr, 0); } static inline __attribute__((always_inline)) void irq_set_noprobe(unsigned int irq) { irq_modify_status(irq, 0, IRQ_NOPROBE); } static inline __attribute__((always_inline)) void irq_set_probe(unsigned int irq) { irq_modify_status(irq, IRQ_NOPROBE, 0); } static inline __attribute__((always_inline)) void irq_set_nothread(unsigned int irq) { irq_modify_status(irq, 0, IRQ_NOTHREAD); } static inline __attribute__((always_inline)) void irq_set_thread(unsigned int irq) { irq_modify_status(irq, IRQ_NOTHREAD, 0); } static inline __attribute__((always_inline)) void irq_set_nested_thread(unsigned int irq, bool nest) { if (__builtin_constant_p(((nest))) ? !!((nest)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/irq.h", .line = 476, }; ______r = !!((nest)); ______f.miss_hit[______r]++; ______r; })) irq_set_status_flags(irq, IRQ_NESTED_THREAD); else irq_clear_status_flags(irq, IRQ_NESTED_THREAD); } extern unsigned int create_irq_nr(unsigned int irq_want, int node); extern int create_irq(void); extern void destroy_irq(unsigned int irq); extern void dynamic_irq_cleanup(unsigned int irq); static inline __attribute__((always_inline)) void dynamic_irq_init(unsigned int irq) { dynamic_irq_cleanup(irq); } extern int irq_set_chip(unsigned int irq, struct irq_chip *chip); extern int irq_set_handler_data(unsigned int irq, void *data); extern int irq_set_chip_data(unsigned int irq, void *data); extern int irq_set_irq_type(unsigned int irq, unsigned int type); extern int irq_set_msi_desc(unsigned int irq, struct msi_desc *entry); extern struct irq_data *irq_get_irq_data(unsigned int irq); static inline __attribute__((always_inline)) struct irq_chip *irq_get_chip(unsigned int irq) { struct irq_data *d = irq_get_irq_data(irq); return d ? d->chip : ((void *)0); } static inline __attribute__((always_inline)) struct irq_chip *irq_data_get_irq_chip(struct irq_data *d) { return d->chip; } static inline __attribute__((always_inline)) void *irq_get_chip_data(unsigned int irq) { struct irq_data *d = irq_get_irq_data(irq); return d ? d->chip_data : ((void *)0); } static inline __attribute__((always_inline)) void *irq_data_get_irq_chip_data(struct irq_data *d) { return d->chip_data; } static inline __attribute__((always_inline)) void *irq_get_handler_data(unsigned int irq) { struct irq_data *d = irq_get_irq_data(irq); return d ? d->handler_data : ((void *)0); } static inline __attribute__((always_inline)) void *irq_data_get_irq_handler_data(struct irq_data *d) { return d->handler_data; } static inline __attribute__((always_inline)) struct msi_desc *irq_get_msi_desc(unsigned int irq) { struct irq_data *d = irq_get_irq_data(irq); return d ? d->msi_desc : ((void *)0); } static inline __attribute__((always_inline)) struct msi_desc *irq_data_get_msi(struct irq_data *d) { return d->msi_desc; } int irq_alloc_descs(int irq, unsigned int from, unsigned int cnt, int node); void irq_free_descs(unsigned int irq, unsigned int cnt); int irq_reserve_irqs(unsigned int from, unsigned int cnt); static inline __attribute__((always_inline)) int irq_alloc_desc(int node) { return irq_alloc_descs(-1, 0, 1, node); } static inline __attribute__((always_inline)) int irq_alloc_desc_at(unsigned int at, int node) { return irq_alloc_descs(at, at, 1, node); } static inline __attribute__((always_inline)) int irq_alloc_desc_from(unsigned int from, int node) { return irq_alloc_descs(-1, from, 1, node); } static inline __attribute__((always_inline)) void irq_free_desc(unsigned int irq) { irq_free_descs(irq, 1); } static inline __attribute__((always_inline)) int irq_reserve_irq(unsigned int irq) { return irq_reserve_irqs(irq, 1); } struct irq_chip_regs { unsigned long enable; unsigned long disable; unsigned long mask; unsigned long ack; unsigned long eoi; unsigned long type; unsigned long polarity; }; struct irq_chip_type { struct irq_chip chip; struct irq_chip_regs regs; irq_flow_handler_t handler; u32 type; }; struct irq_chip_generic { raw_spinlock_t lock; void *reg_base; unsigned int irq_base; unsigned int irq_cnt; u32 mask_cache; u32 type_cache; u32 polarity_cache; u32 wake_enabled; u32 wake_active; unsigned int num_ct; void *private; struct list_head list; struct irq_chip_type chip_types[0]; }; enum irq_gc_flags { IRQ_GC_INIT_MASK_CACHE = 1 << 0, IRQ_GC_INIT_NESTED_LOCK = 1 << 1, }; void irq_gc_noop(struct irq_data *d); void irq_gc_mask_disable_reg(struct irq_data *d); void irq_gc_mask_set_bit(struct irq_data *d); void irq_gc_mask_clr_bit(struct irq_data *d); void irq_gc_unmask_enable_reg(struct irq_data *d); void irq_gc_ack_set_bit(struct irq_data *d); void irq_gc_ack_clr_bit(struct irq_data *d); void irq_gc_mask_disable_reg_and_ack(struct irq_data *d); void irq_gc_eoi(struct irq_data *d); int irq_gc_set_wake(struct irq_data *d, unsigned int on); struct irq_chip_generic * irq_alloc_generic_chip(const char *name, int nr_ct, unsigned int irq_base, void *reg_base, irq_flow_handler_t handler); void irq_setup_generic_chip(struct irq_chip_generic *gc, u32 msk, enum irq_gc_flags flags, unsigned int clr, unsigned int set); int irq_setup_alt_chip(struct irq_data *d, unsigned int type); void irq_remove_generic_chip(struct irq_chip_generic *gc, u32 msk, unsigned int clr, unsigned int set); static inline __attribute__((always_inline)) struct irq_chip_type *irq_data_get_chip_type(struct irq_data *d) { return ({ const typeof( ((struct irq_chip_type *)0)->chip ) *__mptr = (d->chip); (struct irq_chip_type *)( (char *)__mptr - __builtin_offsetof(struct irq_chip_type,chip) );}); } static inline __attribute__((always_inline)) void irq_gc_lock(struct irq_chip_generic *gc) { _raw_spin_lock(&gc->lock); } static inline __attribute__((always_inline)) void irq_gc_unlock(struct irq_chip_generic *gc) { _raw_spin_unlock(&gc->lock); } typedef struct { unsigned int __softirq_pending; unsigned int __nmi_count; unsigned int irq0_irqs; unsigned int apic_timer_irqs; unsigned int irq_spurious_count; unsigned int x86_platform_ipis; unsigned int apic_perf_irqs; unsigned int apic_irq_work_irqs; unsigned int irq_resched_count; unsigned int irq_call_count; unsigned int irq_tlb_count; unsigned int irq_thermal_count; unsigned int irq_threshold_count; } __attribute__((__aligned__((1 << (6))))) irq_cpustat_t; extern __attribute__((section(".data..percpu" ""))) __typeof__(irq_cpustat_t) irq_stat __attribute__((__aligned__((1 << (6))))); extern void ack_bad_irq(unsigned int irq); extern u64 arch_irq_stat_cpu(unsigned int cpu); extern u64 arch_irq_stat(void); extern void synchronize_irq(unsigned int irq); struct task_struct; extern void account_system_vtime(struct task_struct *tsk); extern void irq_enter(void); extern void irq_exit(void); struct stat { unsigned long st_dev; unsigned long st_ino; unsigned short st_mode; unsigned short st_nlink; unsigned short st_uid; unsigned short st_gid; unsigned long st_rdev; unsigned long st_size; unsigned long st_blksize; unsigned long st_blocks; unsigned long st_atime; unsigned long st_atime_nsec; unsigned long st_mtime; unsigned long st_mtime_nsec; unsigned long st_ctime; unsigned long st_ctime_nsec; unsigned long __unused4; unsigned long __unused5; }; struct stat64 { unsigned long long st_dev; unsigned char __pad0[4]; unsigned long __st_ino; unsigned int st_mode; unsigned int st_nlink; unsigned long st_uid; unsigned long st_gid; unsigned long long st_rdev; unsigned char __pad3[4]; long long st_size; unsigned long st_blksize; unsigned long long st_blocks; unsigned long st_atime; unsigned long st_atime_nsec; unsigned long st_mtime; unsigned int st_mtime_nsec; unsigned long st_ctime; unsigned long st_ctime_nsec; unsigned long long st_ino; }; struct __old_kernel_stat { unsigned short st_dev; unsigned short st_ino; unsigned short st_mode; unsigned short st_nlink; unsigned short st_uid; unsigned short st_gid; unsigned short st_rdev; unsigned long st_size; unsigned long st_atime; unsigned long st_mtime; unsigned long st_ctime; }; struct kstat { u64 ino; dev_t dev; umode_t mode; unsigned int nlink; uid_t uid; gid_t gid; dev_t rdev; loff_t size; struct timespec atime; struct timespec mtime; struct timespec ctime; unsigned long blksize; unsigned long long blocks; }; struct completion; struct __sysctl_args { int *name; int nlen; void *oldval; size_t *oldlenp; void *newval; size_t newlen; unsigned long __unused[4]; }; enum { CTL_KERN=1, CTL_VM=2, CTL_NET=3, CTL_PROC=4, CTL_FS=5, CTL_DEBUG=6, CTL_DEV=7, CTL_BUS=8, CTL_ABI=9, CTL_CPU=10, CTL_ARLAN=254, CTL_S390DBF=5677, CTL_SUNRPC=7249, CTL_PM=9899, CTL_FRV=9898, }; enum { CTL_BUS_ISA=1 }; enum { INOTIFY_MAX_USER_INSTANCES=1, INOTIFY_MAX_USER_WATCHES=2, INOTIFY_MAX_QUEUED_EVENTS=3 }; enum { KERN_OSTYPE=1, KERN_OSRELEASE=2, KERN_OSREV=3, KERN_VERSION=4, KERN_SECUREMASK=5, KERN_PROF=6, KERN_NODENAME=7, KERN_DOMAINNAME=8, KERN_PANIC=15, KERN_REALROOTDEV=16, KERN_SPARC_REBOOT=21, KERN_CTLALTDEL=22, KERN_PRINTK=23, KERN_NAMETRANS=24, KERN_PPC_HTABRECLAIM=25, KERN_PPC_ZEROPAGED=26, KERN_PPC_POWERSAVE_NAP=27, KERN_MODPROBE=28, KERN_SG_BIG_BUFF=29, KERN_ACCT=30, KERN_PPC_L2CR=31, KERN_RTSIGNR=32, KERN_RTSIGMAX=33, KERN_SHMMAX=34, KERN_MSGMAX=35, KERN_MSGMNB=36, KERN_MSGPOOL=37, KERN_SYSRQ=38, KERN_MAX_THREADS=39, KERN_RANDOM=40, KERN_SHMALL=41, KERN_MSGMNI=42, KERN_SEM=43, KERN_SPARC_STOP_A=44, KERN_SHMMNI=45, KERN_OVERFLOWUID=46, KERN_OVERFLOWGID=47, KERN_SHMPATH=48, KERN_HOTPLUG=49, KERN_IEEE_EMULATION_WARNINGS=50, KERN_S390_USER_DEBUG_LOGGING=51, KERN_CORE_USES_PID=52, KERN_TAINTED=53, KERN_CADPID=54, KERN_PIDMAX=55, KERN_CORE_PATTERN=56, KERN_PANIC_ON_OOPS=57, KERN_HPPA_PWRSW=58, KERN_HPPA_UNALIGNED=59, KERN_PRINTK_RATELIMIT=60, KERN_PRINTK_RATELIMIT_BURST=61, KERN_PTY=62, KERN_NGROUPS_MAX=63, KERN_SPARC_SCONS_PWROFF=64, KERN_HZ_TIMER=65, KERN_UNKNOWN_NMI_PANIC=66, KERN_BOOTLOADER_TYPE=67, KERN_RANDOMIZE=68, KERN_SETUID_DUMPABLE=69, KERN_SPIN_RETRY=70, KERN_ACPI_VIDEO_FLAGS=71, KERN_IA64_UNALIGNED=72, KERN_COMPAT_LOG=73, KERN_MAX_LOCK_DEPTH=74, KERN_NMI_WATCHDOG=75, KERN_PANIC_ON_NMI=76, }; enum { VM_UNUSED1=1, VM_UNUSED2=2, VM_UNUSED3=3, VM_UNUSED4=4, VM_OVERCOMMIT_MEMORY=5, VM_UNUSED5=6, VM_UNUSED7=7, VM_UNUSED8=8, VM_UNUSED9=9, VM_PAGE_CLUSTER=10, VM_DIRTY_BACKGROUND=11, VM_DIRTY_RATIO=12, VM_DIRTY_WB_CS=13, VM_DIRTY_EXPIRE_CS=14, VM_NR_PDFLUSH_THREADS=15, VM_OVERCOMMIT_RATIO=16, VM_PAGEBUF=17, VM_HUGETLB_PAGES=18, VM_SWAPPINESS=19, VM_LOWMEM_RESERVE_RATIO=20, VM_MIN_FREE_KBYTES=21, VM_MAX_MAP_COUNT=22, VM_LAPTOP_MODE=23, VM_BLOCK_DUMP=24, VM_HUGETLB_GROUP=25, VM_VFS_CACHE_PRESSURE=26, VM_LEGACY_VA_LAYOUT=27, VM_SWAP_TOKEN_TIMEOUT=28, VM_DROP_PAGECACHE=29, VM_PERCPU_PAGELIST_FRACTION=30, VM_ZONE_RECLAIM_MODE=31, VM_MIN_UNMAPPED=32, VM_PANIC_ON_OOM=33, VM_VDSO_ENABLED=34, VM_MIN_SLAB=35, }; enum { NET_CORE=1, NET_ETHER=2, NET_802=3, NET_UNIX=4, NET_IPV4=5, NET_IPX=6, NET_ATALK=7, NET_NETROM=8, NET_AX25=9, NET_BRIDGE=10, NET_ROSE=11, NET_IPV6=12, NET_X25=13, NET_TR=14, NET_DECNET=15, NET_ECONET=16, NET_SCTP=17, NET_LLC=18, NET_NETFILTER=19, NET_DCCP=20, NET_IRDA=412, }; enum { RANDOM_POOLSIZE=1, RANDOM_ENTROPY_COUNT=2, RANDOM_READ_THRESH=3, RANDOM_WRITE_THRESH=4, RANDOM_BOOT_ID=5, RANDOM_UUID=6 }; enum { PTY_MAX=1, PTY_NR=2 }; enum { BUS_ISA_MEM_BASE=1, BUS_ISA_PORT_BASE=2, BUS_ISA_PORT_SHIFT=3 }; enum { NET_CORE_WMEM_MAX=1, NET_CORE_RMEM_MAX=2, NET_CORE_WMEM_DEFAULT=3, NET_CORE_RMEM_DEFAULT=4, NET_CORE_MAX_BACKLOG=6, NET_CORE_FASTROUTE=7, NET_CORE_MSG_COST=8, NET_CORE_MSG_BURST=9, NET_CORE_OPTMEM_MAX=10, NET_CORE_HOT_LIST_LENGTH=11, NET_CORE_DIVERT_VERSION=12, NET_CORE_NO_CONG_THRESH=13, NET_CORE_NO_CONG=14, NET_CORE_LO_CONG=15, NET_CORE_MOD_CONG=16, NET_CORE_DEV_WEIGHT=17, NET_CORE_SOMAXCONN=18, NET_CORE_BUDGET=19, NET_CORE_AEVENT_ETIME=20, NET_CORE_AEVENT_RSEQTH=21, NET_CORE_WARNINGS=22, }; enum { NET_UNIX_DESTROY_DELAY=1, NET_UNIX_DELETE_DELAY=2, NET_UNIX_MAX_DGRAM_QLEN=3, }; enum { NET_NF_CONNTRACK_MAX=1, NET_NF_CONNTRACK_TCP_TIMEOUT_SYN_SENT=2, NET_NF_CONNTRACK_TCP_TIMEOUT_SYN_RECV=3, NET_NF_CONNTRACK_TCP_TIMEOUT_ESTABLISHED=4, NET_NF_CONNTRACK_TCP_TIMEOUT_FIN_WAIT=5, NET_NF_CONNTRACK_TCP_TIMEOUT_CLOSE_WAIT=6, NET_NF_CONNTRACK_TCP_TIMEOUT_LAST_ACK=7, NET_NF_CONNTRACK_TCP_TIMEOUT_TIME_WAIT=8, NET_NF_CONNTRACK_TCP_TIMEOUT_CLOSE=9, NET_NF_CONNTRACK_UDP_TIMEOUT=10, NET_NF_CONNTRACK_UDP_TIMEOUT_STREAM=11, NET_NF_CONNTRACK_ICMP_TIMEOUT=12, NET_NF_CONNTRACK_GENERIC_TIMEOUT=13, NET_NF_CONNTRACK_BUCKETS=14, NET_NF_CONNTRACK_LOG_INVALID=15, NET_NF_CONNTRACK_TCP_TIMEOUT_MAX_RETRANS=16, NET_NF_CONNTRACK_TCP_LOOSE=17, NET_NF_CONNTRACK_TCP_BE_LIBERAL=18, NET_NF_CONNTRACK_TCP_MAX_RETRANS=19, NET_NF_CONNTRACK_SCTP_TIMEOUT_CLOSED=20, NET_NF_CONNTRACK_SCTP_TIMEOUT_COOKIE_WAIT=21, NET_NF_CONNTRACK_SCTP_TIMEOUT_COOKIE_ECHOED=22, NET_NF_CONNTRACK_SCTP_TIMEOUT_ESTABLISHED=23, NET_NF_CONNTRACK_SCTP_TIMEOUT_SHUTDOWN_SENT=24, NET_NF_CONNTRACK_SCTP_TIMEOUT_SHUTDOWN_RECD=25, NET_NF_CONNTRACK_SCTP_TIMEOUT_SHUTDOWN_ACK_SENT=26, NET_NF_CONNTRACK_COUNT=27, NET_NF_CONNTRACK_ICMPV6_TIMEOUT=28, NET_NF_CONNTRACK_FRAG6_TIMEOUT=29, NET_NF_CONNTRACK_FRAG6_LOW_THRESH=30, NET_NF_CONNTRACK_FRAG6_HIGH_THRESH=31, NET_NF_CONNTRACK_CHECKSUM=32, }; enum { NET_IPV4_FORWARD=8, NET_IPV4_DYNADDR=9, NET_IPV4_CONF=16, NET_IPV4_NEIGH=17, NET_IPV4_ROUTE=18, NET_IPV4_FIB_HASH=19, NET_IPV4_NETFILTER=20, NET_IPV4_TCP_TIMESTAMPS=33, NET_IPV4_TCP_WINDOW_SCALING=34, NET_IPV4_TCP_SACK=35, NET_IPV4_TCP_RETRANS_COLLAPSE=36, NET_IPV4_DEFAULT_TTL=37, NET_IPV4_AUTOCONFIG=38, NET_IPV4_NO_PMTU_DISC=39, NET_IPV4_TCP_SYN_RETRIES=40, NET_IPV4_IPFRAG_HIGH_THRESH=41, NET_IPV4_IPFRAG_LOW_THRESH=42, NET_IPV4_IPFRAG_TIME=43, NET_IPV4_TCP_MAX_KA_PROBES=44, NET_IPV4_TCP_KEEPALIVE_TIME=45, NET_IPV4_TCP_KEEPALIVE_PROBES=46, NET_IPV4_TCP_RETRIES1=47, NET_IPV4_TCP_RETRIES2=48, NET_IPV4_TCP_FIN_TIMEOUT=49, NET_IPV4_IP_MASQ_DEBUG=50, NET_TCP_SYNCOOKIES=51, NET_TCP_STDURG=52, NET_TCP_RFC1337=53, NET_TCP_SYN_TAILDROP=54, NET_TCP_MAX_SYN_BACKLOG=55, NET_IPV4_LOCAL_PORT_RANGE=56, NET_IPV4_ICMP_ECHO_IGNORE_ALL=57, NET_IPV4_ICMP_ECHO_IGNORE_BROADCASTS=58, NET_IPV4_ICMP_SOURCEQUENCH_RATE=59, NET_IPV4_ICMP_DESTUNREACH_RATE=60, NET_IPV4_ICMP_TIMEEXCEED_RATE=61, NET_IPV4_ICMP_PARAMPROB_RATE=62, NET_IPV4_ICMP_ECHOREPLY_RATE=63, NET_IPV4_ICMP_IGNORE_BOGUS_ERROR_RESPONSES=64, NET_IPV4_IGMP_MAX_MEMBERSHIPS=65, NET_TCP_TW_RECYCLE=66, NET_IPV4_ALWAYS_DEFRAG=67, NET_IPV4_TCP_KEEPALIVE_INTVL=68, NET_IPV4_INET_PEER_THRESHOLD=69, NET_IPV4_INET_PEER_MINTTL=70, NET_IPV4_INET_PEER_MAXTTL=71, NET_IPV4_INET_PEER_GC_MINTIME=72, NET_IPV4_INET_PEER_GC_MAXTIME=73, NET_TCP_ORPHAN_RETRIES=74, NET_TCP_ABORT_ON_OVERFLOW=75, NET_TCP_SYNACK_RETRIES=76, NET_TCP_MAX_ORPHANS=77, NET_TCP_MAX_TW_BUCKETS=78, NET_TCP_FACK=79, NET_TCP_REORDERING=80, NET_TCP_ECN=81, NET_TCP_DSACK=82, NET_TCP_MEM=83, NET_TCP_WMEM=84, NET_TCP_RMEM=85, NET_TCP_APP_WIN=86, NET_TCP_ADV_WIN_SCALE=87, NET_IPV4_NONLOCAL_BIND=88, NET_IPV4_ICMP_RATELIMIT=89, NET_IPV4_ICMP_RATEMASK=90, NET_TCP_TW_REUSE=91, NET_TCP_FRTO=92, NET_TCP_LOW_LATENCY=93, NET_IPV4_IPFRAG_SECRET_INTERVAL=94, NET_IPV4_IGMP_MAX_MSF=96, NET_TCP_NO_METRICS_SAVE=97, NET_TCP_DEFAULT_WIN_SCALE=105, NET_TCP_MODERATE_RCVBUF=106, NET_TCP_TSO_WIN_DIVISOR=107, NET_TCP_BIC_BETA=108, NET_IPV4_ICMP_ERRORS_USE_INBOUND_IFADDR=109, NET_TCP_CONG_CONTROL=110, NET_TCP_ABC=111, NET_IPV4_IPFRAG_MAX_DIST=112, NET_TCP_MTU_PROBING=113, NET_TCP_BASE_MSS=114, NET_IPV4_TCP_WORKAROUND_SIGNED_WINDOWS=115, NET_TCP_DMA_COPYBREAK=116, NET_TCP_SLOW_START_AFTER_IDLE=117, NET_CIPSOV4_CACHE_ENABLE=118, NET_CIPSOV4_CACHE_BUCKET_SIZE=119, NET_CIPSOV4_RBM_OPTFMT=120, NET_CIPSOV4_RBM_STRICTVALID=121, NET_TCP_AVAIL_CONG_CONTROL=122, NET_TCP_ALLOWED_CONG_CONTROL=123, NET_TCP_MAX_SSTHRESH=124, NET_TCP_FRTO_RESPONSE=125, }; enum { NET_IPV4_ROUTE_FLUSH=1, NET_IPV4_ROUTE_MIN_DELAY=2, NET_IPV4_ROUTE_MAX_DELAY=3, NET_IPV4_ROUTE_GC_THRESH=4, NET_IPV4_ROUTE_MAX_SIZE=5, NET_IPV4_ROUTE_GC_MIN_INTERVAL=6, NET_IPV4_ROUTE_GC_TIMEOUT=7, NET_IPV4_ROUTE_GC_INTERVAL=8, NET_IPV4_ROUTE_REDIRECT_LOAD=9, NET_IPV4_ROUTE_REDIRECT_NUMBER=10, NET_IPV4_ROUTE_REDIRECT_SILENCE=11, NET_IPV4_ROUTE_ERROR_COST=12, NET_IPV4_ROUTE_ERROR_BURST=13, NET_IPV4_ROUTE_GC_ELASTICITY=14, NET_IPV4_ROUTE_MTU_EXPIRES=15, NET_IPV4_ROUTE_MIN_PMTU=16, NET_IPV4_ROUTE_MIN_ADVMSS=17, NET_IPV4_ROUTE_SECRET_INTERVAL=18, NET_IPV4_ROUTE_GC_MIN_INTERVAL_MS=19, }; enum { NET_PROTO_CONF_ALL=-2, NET_PROTO_CONF_DEFAULT=-3 }; enum { NET_IPV4_CONF_FORWARDING=1, NET_IPV4_CONF_MC_FORWARDING=2, NET_IPV4_CONF_PROXY_ARP=3, NET_IPV4_CONF_ACCEPT_REDIRECTS=4, NET_IPV4_CONF_SECURE_REDIRECTS=5, NET_IPV4_CONF_SEND_REDIRECTS=6, NET_IPV4_CONF_SHARED_MEDIA=7, NET_IPV4_CONF_RP_FILTER=8, NET_IPV4_CONF_ACCEPT_SOURCE_ROUTE=9, NET_IPV4_CONF_BOOTP_RELAY=10, NET_IPV4_CONF_LOG_MARTIANS=11, NET_IPV4_CONF_TAG=12, NET_IPV4_CONF_ARPFILTER=13, NET_IPV4_CONF_MEDIUM_ID=14, NET_IPV4_CONF_NOXFRM=15, NET_IPV4_CONF_NOPOLICY=16, NET_IPV4_CONF_FORCE_IGMP_VERSION=17, NET_IPV4_CONF_ARP_ANNOUNCE=18, NET_IPV4_CONF_ARP_IGNORE=19, NET_IPV4_CONF_PROMOTE_SECONDARIES=20, NET_IPV4_CONF_ARP_ACCEPT=21, NET_IPV4_CONF_ARP_NOTIFY=22, }; enum { NET_IPV4_NF_CONNTRACK_MAX=1, NET_IPV4_NF_CONNTRACK_TCP_TIMEOUT_SYN_SENT=2, NET_IPV4_NF_CONNTRACK_TCP_TIMEOUT_SYN_RECV=3, NET_IPV4_NF_CONNTRACK_TCP_TIMEOUT_ESTABLISHED=4, NET_IPV4_NF_CONNTRACK_TCP_TIMEOUT_FIN_WAIT=5, NET_IPV4_NF_CONNTRACK_TCP_TIMEOUT_CLOSE_WAIT=6, NET_IPV4_NF_CONNTRACK_TCP_TIMEOUT_LAST_ACK=7, NET_IPV4_NF_CONNTRACK_TCP_TIMEOUT_TIME_WAIT=8, NET_IPV4_NF_CONNTRACK_TCP_TIMEOUT_CLOSE=9, NET_IPV4_NF_CONNTRACK_UDP_TIMEOUT=10, NET_IPV4_NF_CONNTRACK_UDP_TIMEOUT_STREAM=11, NET_IPV4_NF_CONNTRACK_ICMP_TIMEOUT=12, NET_IPV4_NF_CONNTRACK_GENERIC_TIMEOUT=13, NET_IPV4_NF_CONNTRACK_BUCKETS=14, NET_IPV4_NF_CONNTRACK_LOG_INVALID=15, NET_IPV4_NF_CONNTRACK_TCP_TIMEOUT_MAX_RETRANS=16, NET_IPV4_NF_CONNTRACK_TCP_LOOSE=17, NET_IPV4_NF_CONNTRACK_TCP_BE_LIBERAL=18, NET_IPV4_NF_CONNTRACK_TCP_MAX_RETRANS=19, NET_IPV4_NF_CONNTRACK_SCTP_TIMEOUT_CLOSED=20, NET_IPV4_NF_CONNTRACK_SCTP_TIMEOUT_COOKIE_WAIT=21, NET_IPV4_NF_CONNTRACK_SCTP_TIMEOUT_COOKIE_ECHOED=22, NET_IPV4_NF_CONNTRACK_SCTP_TIMEOUT_ESTABLISHED=23, NET_IPV4_NF_CONNTRACK_SCTP_TIMEOUT_SHUTDOWN_SENT=24, NET_IPV4_NF_CONNTRACK_SCTP_TIMEOUT_SHUTDOWN_RECD=25, NET_IPV4_NF_CONNTRACK_SCTP_TIMEOUT_SHUTDOWN_ACK_SENT=26, NET_IPV4_NF_CONNTRACK_COUNT=27, NET_IPV4_NF_CONNTRACK_CHECKSUM=28, }; enum { NET_IPV6_CONF=16, NET_IPV6_NEIGH=17, NET_IPV6_ROUTE=18, NET_IPV6_ICMP=19, NET_IPV6_BINDV6ONLY=20, NET_IPV6_IP6FRAG_HIGH_THRESH=21, NET_IPV6_IP6FRAG_LOW_THRESH=22, NET_IPV6_IP6FRAG_TIME=23, NET_IPV6_IP6FRAG_SECRET_INTERVAL=24, NET_IPV6_MLD_MAX_MSF=25, }; enum { NET_IPV6_ROUTE_FLUSH=1, NET_IPV6_ROUTE_GC_THRESH=2, NET_IPV6_ROUTE_MAX_SIZE=3, NET_IPV6_ROUTE_GC_MIN_INTERVAL=4, NET_IPV6_ROUTE_GC_TIMEOUT=5, NET_IPV6_ROUTE_GC_INTERVAL=6, NET_IPV6_ROUTE_GC_ELASTICITY=7, NET_IPV6_ROUTE_MTU_EXPIRES=8, NET_IPV6_ROUTE_MIN_ADVMSS=9, NET_IPV6_ROUTE_GC_MIN_INTERVAL_MS=10 }; enum { NET_IPV6_FORWARDING=1, NET_IPV6_HOP_LIMIT=2, NET_IPV6_MTU=3, NET_IPV6_ACCEPT_RA=4, NET_IPV6_ACCEPT_REDIRECTS=5, NET_IPV6_AUTOCONF=6, NET_IPV6_DAD_TRANSMITS=7, NET_IPV6_RTR_SOLICITS=8, NET_IPV6_RTR_SOLICIT_INTERVAL=9, NET_IPV6_RTR_SOLICIT_DELAY=10, NET_IPV6_USE_TEMPADDR=11, NET_IPV6_TEMP_VALID_LFT=12, NET_IPV6_TEMP_PREFERED_LFT=13, NET_IPV6_REGEN_MAX_RETRY=14, NET_IPV6_MAX_DESYNC_FACTOR=15, NET_IPV6_MAX_ADDRESSES=16, NET_IPV6_FORCE_MLD_VERSION=17, NET_IPV6_ACCEPT_RA_DEFRTR=18, NET_IPV6_ACCEPT_RA_PINFO=19, NET_IPV6_ACCEPT_RA_RTR_PREF=20, NET_IPV6_RTR_PROBE_INTERVAL=21, NET_IPV6_ACCEPT_RA_RT_INFO_MAX_PLEN=22, NET_IPV6_PROXY_NDP=23, NET_IPV6_ACCEPT_SOURCE_ROUTE=25, __NET_IPV6_MAX }; enum { NET_IPV6_ICMP_RATELIMIT=1 }; enum { NET_NEIGH_MCAST_SOLICIT=1, NET_NEIGH_UCAST_SOLICIT=2, NET_NEIGH_APP_SOLICIT=3, NET_NEIGH_RETRANS_TIME=4, NET_NEIGH_REACHABLE_TIME=5, NET_NEIGH_DELAY_PROBE_TIME=6, NET_NEIGH_GC_STALE_TIME=7, NET_NEIGH_UNRES_QLEN=8, NET_NEIGH_PROXY_QLEN=9, NET_NEIGH_ANYCAST_DELAY=10, NET_NEIGH_PROXY_DELAY=11, NET_NEIGH_LOCKTIME=12, NET_NEIGH_GC_INTERVAL=13, NET_NEIGH_GC_THRESH1=14, NET_NEIGH_GC_THRESH2=15, NET_NEIGH_GC_THRESH3=16, NET_NEIGH_RETRANS_TIME_MS=17, NET_NEIGH_REACHABLE_TIME_MS=18, }; enum { NET_DCCP_DEFAULT=1, }; enum { NET_IPX_PPROP_BROADCASTING=1, NET_IPX_FORWARDING=2 }; enum { NET_LLC2=1, NET_LLC_STATION=2, }; enum { NET_LLC2_TIMEOUT=1, }; enum { NET_LLC_STATION_ACK_TIMEOUT=1, }; enum { NET_LLC2_ACK_TIMEOUT=1, NET_LLC2_P_TIMEOUT=2, NET_LLC2_REJ_TIMEOUT=3, NET_LLC2_BUSY_TIMEOUT=4, }; enum { NET_ATALK_AARP_EXPIRY_TIME=1, NET_ATALK_AARP_TICK_TIME=2, NET_ATALK_AARP_RETRANSMIT_LIMIT=3, NET_ATALK_AARP_RESOLVE_TIME=4 }; enum { NET_NETROM_DEFAULT_PATH_QUALITY=1, NET_NETROM_OBSOLESCENCE_COUNT_INITIALISER=2, NET_NETROM_NETWORK_TTL_INITIALISER=3, NET_NETROM_TRANSPORT_TIMEOUT=4, NET_NETROM_TRANSPORT_MAXIMUM_TRIES=5, NET_NETROM_TRANSPORT_ACKNOWLEDGE_DELAY=6, NET_NETROM_TRANSPORT_BUSY_DELAY=7, NET_NETROM_TRANSPORT_REQUESTED_WINDOW_SIZE=8, NET_NETROM_TRANSPORT_NO_ACTIVITY_TIMEOUT=9, NET_NETROM_ROUTING_CONTROL=10, NET_NETROM_LINK_FAILS_COUNT=11, NET_NETROM_RESET=12 }; enum { NET_AX25_IP_DEFAULT_MODE=1, NET_AX25_DEFAULT_MODE=2, NET_AX25_BACKOFF_TYPE=3, NET_AX25_CONNECT_MODE=4, NET_AX25_STANDARD_WINDOW=5, NET_AX25_EXTENDED_WINDOW=6, NET_AX25_T1_TIMEOUT=7, NET_AX25_T2_TIMEOUT=8, NET_AX25_T3_TIMEOUT=9, NET_AX25_IDLE_TIMEOUT=10, NET_AX25_N2=11, NET_AX25_PACLEN=12, NET_AX25_PROTOCOL=13, NET_AX25_DAMA_SLAVE_TIMEOUT=14 }; enum { NET_ROSE_RESTART_REQUEST_TIMEOUT=1, NET_ROSE_CALL_REQUEST_TIMEOUT=2, NET_ROSE_RESET_REQUEST_TIMEOUT=3, NET_ROSE_CLEAR_REQUEST_TIMEOUT=4, NET_ROSE_ACK_HOLD_BACK_TIMEOUT=5, NET_ROSE_ROUTING_CONTROL=6, NET_ROSE_LINK_FAIL_TIMEOUT=7, NET_ROSE_MAX_VCS=8, NET_ROSE_WINDOW_SIZE=9, NET_ROSE_NO_ACTIVITY_TIMEOUT=10 }; enum { NET_X25_RESTART_REQUEST_TIMEOUT=1, NET_X25_CALL_REQUEST_TIMEOUT=2, NET_X25_RESET_REQUEST_TIMEOUT=3, NET_X25_CLEAR_REQUEST_TIMEOUT=4, NET_X25_ACK_HOLD_BACK_TIMEOUT=5, NET_X25_FORWARD=6 }; enum { NET_TR_RIF_TIMEOUT=1 }; enum { NET_DECNET_NODE_TYPE = 1, NET_DECNET_NODE_ADDRESS = 2, NET_DECNET_NODE_NAME = 3, NET_DECNET_DEFAULT_DEVICE = 4, NET_DECNET_TIME_WAIT = 5, NET_DECNET_DN_COUNT = 6, NET_DECNET_DI_COUNT = 7, NET_DECNET_DR_COUNT = 8, NET_DECNET_DST_GC_INTERVAL = 9, NET_DECNET_CONF = 10, NET_DECNET_NO_FC_MAX_CWND = 11, NET_DECNET_MEM = 12, NET_DECNET_RMEM = 13, NET_DECNET_WMEM = 14, NET_DECNET_DEBUG_LEVEL = 255 }; enum { NET_DECNET_CONF_LOOPBACK = -2, NET_DECNET_CONF_DDCMP = -3, NET_DECNET_CONF_PPP = -4, NET_DECNET_CONF_X25 = -5, NET_DECNET_CONF_GRE = -6, NET_DECNET_CONF_ETHER = -7 }; enum { NET_DECNET_CONF_DEV_PRIORITY = 1, NET_DECNET_CONF_DEV_T1 = 2, NET_DECNET_CONF_DEV_T2 = 3, NET_DECNET_CONF_DEV_T3 = 4, NET_DECNET_CONF_DEV_FORWARDING = 5, NET_DECNET_CONF_DEV_BLKSIZE = 6, NET_DECNET_CONF_DEV_STATE = 7 }; enum { NET_SCTP_RTO_INITIAL = 1, NET_SCTP_RTO_MIN = 2, NET_SCTP_RTO_MAX = 3, NET_SCTP_RTO_ALPHA = 4, NET_SCTP_RTO_BETA = 5, NET_SCTP_VALID_COOKIE_LIFE = 6, NET_SCTP_ASSOCIATION_MAX_RETRANS = 7, NET_SCTP_PATH_MAX_RETRANS = 8, NET_SCTP_MAX_INIT_RETRANSMITS = 9, NET_SCTP_HB_INTERVAL = 10, NET_SCTP_PRESERVE_ENABLE = 11, NET_SCTP_MAX_BURST = 12, NET_SCTP_ADDIP_ENABLE = 13, NET_SCTP_PRSCTP_ENABLE = 14, NET_SCTP_SNDBUF_POLICY = 15, NET_SCTP_SACK_TIMEOUT = 16, NET_SCTP_RCVBUF_POLICY = 17, }; enum { NET_BRIDGE_NF_CALL_ARPTABLES = 1, NET_BRIDGE_NF_CALL_IPTABLES = 2, NET_BRIDGE_NF_CALL_IP6TABLES = 3, NET_BRIDGE_NF_FILTER_VLAN_TAGGED = 4, NET_BRIDGE_NF_FILTER_PPPOE_TAGGED = 5, }; enum { NET_IRDA_DISCOVERY=1, NET_IRDA_DEVNAME=2, NET_IRDA_DEBUG=3, NET_IRDA_FAST_POLL=4, NET_IRDA_DISCOVERY_SLOTS=5, NET_IRDA_DISCOVERY_TIMEOUT=6, NET_IRDA_SLOT_TIMEOUT=7, NET_IRDA_MAX_BAUD_RATE=8, NET_IRDA_MIN_TX_TURN_TIME=9, NET_IRDA_MAX_TX_DATA_SIZE=10, NET_IRDA_MAX_TX_WINDOW=11, NET_IRDA_MAX_NOREPLY_TIME=12, NET_IRDA_WARN_NOREPLY_TIME=13, NET_IRDA_LAP_KEEPALIVE_TIME=14, }; enum { FS_NRINODE=1, FS_STATINODE=2, FS_MAXINODE=3, FS_NRDQUOT=4, FS_MAXDQUOT=5, FS_NRFILE=6, FS_MAXFILE=7, FS_DENTRY=8, FS_NRSUPER=9, FS_MAXSUPER=10, FS_OVERFLOWUID=11, FS_OVERFLOWGID=12, FS_LEASES=13, FS_DIR_NOTIFY=14, FS_LEASE_TIME=15, FS_DQSTATS=16, FS_XFS=17, FS_AIO_NR=18, FS_AIO_MAX_NR=19, FS_INOTIFY=20, FS_OCFS2=988, }; enum { FS_DQ_LOOKUPS = 1, FS_DQ_DROPS = 2, FS_DQ_READS = 3, FS_DQ_WRITES = 4, FS_DQ_CACHE_HITS = 5, FS_DQ_ALLOCATED = 6, FS_DQ_FREE = 7, FS_DQ_SYNCS = 8, FS_DQ_WARNINGS = 9, }; enum { DEV_CDROM=1, DEV_HWMON=2, DEV_PARPORT=3, DEV_RAID=4, DEV_MAC_HID=5, DEV_SCSI=6, DEV_IPMI=7, }; enum { DEV_CDROM_INFO=1, DEV_CDROM_AUTOCLOSE=2, DEV_CDROM_AUTOEJECT=3, DEV_CDROM_DEBUG=4, DEV_CDROM_LOCK=5, DEV_CDROM_CHECK_MEDIA=6 }; enum { DEV_PARPORT_DEFAULT=-3 }; enum { DEV_RAID_SPEED_LIMIT_MIN=1, DEV_RAID_SPEED_LIMIT_MAX=2 }; enum { DEV_PARPORT_DEFAULT_TIMESLICE=1, DEV_PARPORT_DEFAULT_SPINTIME=2 }; enum { DEV_PARPORT_SPINTIME=1, DEV_PARPORT_BASE_ADDR=2, DEV_PARPORT_IRQ=3, DEV_PARPORT_DMA=4, DEV_PARPORT_MODES=5, DEV_PARPORT_DEVICES=6, DEV_PARPORT_AUTOPROBE=16 }; enum { DEV_PARPORT_DEVICES_ACTIVE=-3, }; enum { DEV_PARPORT_DEVICE_TIMESLICE=1, }; enum { DEV_MAC_HID_KEYBOARD_SENDS_LINUX_KEYCODES=1, DEV_MAC_HID_KEYBOARD_LOCK_KEYCODES=2, DEV_MAC_HID_MOUSE_BUTTON_EMULATION=3, DEV_MAC_HID_MOUSE_BUTTON2_KEYCODE=4, DEV_MAC_HID_MOUSE_BUTTON3_KEYCODE=5, DEV_MAC_HID_ADB_MOUSE_SENDS_KEYCODES=6 }; enum { DEV_SCSI_LOGGING_LEVEL=1, }; enum { DEV_IPMI_POWEROFF_POWERCYCLE=1, }; enum { ABI_DEFHANDLER_COFF=1, ABI_DEFHANDLER_ELF=2, ABI_DEFHANDLER_LCALL7=3, ABI_DEFHANDLER_LIBCSO=4, ABI_TRACE=5, ABI_FAKE_UTSNAME=6, }; extern void rcutorture_record_test_transition(void); extern void rcutorture_record_progress(unsigned long vernum); struct rcu_head { struct rcu_head *next; void (*func)(struct rcu_head *head); }; extern void call_rcu_sched(struct rcu_head *head, void (*func)(struct rcu_head *rcu)); extern void synchronize_sched(void); extern void rcu_barrier_bh(void); extern void rcu_barrier_sched(void); static inline __attribute__((always_inline)) void __rcu_read_lock_bh(void) { local_bh_disable(); } static inline __attribute__((always_inline)) void __rcu_read_unlock_bh(void) { local_bh_enable(); } extern void __rcu_read_lock(void); extern void __rcu_read_unlock(void); void synchronize_rcu(void); extern void rcu_sched_qs(int cpu); extern void rcu_bh_qs(int cpu); extern void rcu_check_callbacks(int cpu, int user); struct notifier_block; static inline __attribute__((always_inline)) void rcu_enter_nohz(void) { } static inline __attribute__((always_inline)) void rcu_exit_nohz(void) { } extern void rcu_init(void); extern void rcu_note_context_switch(int cpu); extern int rcu_needs_cpu(int cpu); extern void rcu_cpu_stall_reset(void); static inline __attribute__((always_inline)) void rcu_virt_note_context_switch(int cpu) { rcu_note_context_switch(cpu); } extern void exit_rcu(void); extern void synchronize_rcu_bh(void); extern void synchronize_sched_expedited(void); extern void synchronize_rcu_expedited(void); static inline __attribute__((always_inline)) void synchronize_rcu_bh_expedited(void) { synchronize_sched_expedited(); } extern void rcu_barrier(void); extern unsigned long rcutorture_testseq; extern unsigned long rcutorture_vernum; extern long rcu_batches_completed(void); extern long rcu_batches_completed_bh(void); extern long rcu_batches_completed_sched(void); extern void rcu_force_quiescent_state(void); extern void rcu_bh_force_quiescent_state(void); extern void rcu_sched_force_quiescent_state(void); static inline __attribute__((always_inline)) int rcu_blocking_is_gp(void) { return cpumask_weight(cpu_online_mask) == 1; } extern void rcu_scheduler_starting(void); extern int rcu_scheduler_active __attribute__((__section__(".data..read_mostly"))); static inline __attribute__((always_inline)) void init_rcu_head_on_stack(struct rcu_head *head) { } static inline __attribute__((always_inline)) void destroy_rcu_head_on_stack(struct rcu_head *head) { } extern struct lockdep_map rcu_lock_map; extern struct lockdep_map rcu_bh_lock_map; extern struct lockdep_map rcu_sched_lock_map; extern int debug_lockdep_rcu_enabled(void); static inline __attribute__((always_inline)) int rcu_read_lock_held(void) { if (__builtin_constant_p(((!debug_lockdep_rcu_enabled()))) ? !!((!debug_lockdep_rcu_enabled())) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/rcupdate.h", .line = 216, }; ______r = !!((!debug_lockdep_rcu_enabled())); ______f.miss_hit[______r]++; ______r; })) return 1; return lock_is_held(&rcu_lock_map); } extern int rcu_read_lock_bh_held(void); static inline __attribute__((always_inline)) int rcu_read_lock_sched_held(void) { int lockdep_opinion = 0; if (__builtin_constant_p(((!debug_lockdep_rcu_enabled()))) ? !!((!debug_lockdep_rcu_enabled())) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/rcupdate.h", .line = 247, }; ______r = !!((!debug_lockdep_rcu_enabled())); ______f.miss_hit[______r]++; ______r; })) return 1; if (__builtin_constant_p(((debug_locks))) ? !!((debug_locks)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/rcupdate.h", .line = 249, }; ______r = !!((debug_locks)); ______f.miss_hit[______r]++; ______r; })) lockdep_opinion = lock_is_held(&rcu_sched_lock_map); return lockdep_opinion || (current_thread_info()->preempt_count) != 0 || ({ unsigned long _flags; do { ({ unsigned long __dummy; typeof(_flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); _flags = arch_local_save_flags(); } while (0); ({ ({ unsigned long __dummy; typeof(_flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); arch_irqs_disabled_flags(_flags); }); }); } static inline __attribute__((always_inline)) void rcu_read_lock(void) { __rcu_read_lock(); (void)0; lock_acquire(&rcu_lock_map, 0, 0, 2, 1, ((void *)0), ({ __label__ __here; __here: (unsigned long)&&__here; })); } static inline __attribute__((always_inline)) void rcu_read_unlock(void) { lock_release(&rcu_lock_map, 1, ({ __label__ __here; __here: (unsigned long)&&__here; })); (void)0; __rcu_read_unlock(); } static inline __attribute__((always_inline)) void rcu_read_lock_bh(void) { __rcu_read_lock_bh(); (void)0; lock_acquire(&rcu_bh_lock_map, 0, 0, 2, 1, ((void *)0), ({ __label__ __here; __here: (unsigned long)&&__here; })); } static inline __attribute__((always_inline)) void rcu_read_unlock_bh(void) { lock_release(&rcu_bh_lock_map, 1, ({ __label__ __here; __here: (unsigned long)&&__here; })); (void)0; __rcu_read_unlock_bh(); } static inline __attribute__((always_inline)) void rcu_read_lock_sched(void) { do { add_preempt_count(1); __asm__ __volatile__("": : :"memory"); } while (0); (void)0; lock_acquire(&rcu_sched_lock_map, 0, 0, 2, 1, ((void *)0), ({ __label__ __here; __here: (unsigned long)&&__here; })); } static inline __attribute__((always_inline)) __attribute__((no_instrument_function)) void rcu_read_lock_sched_notrace(void) { do { do { (current_thread_info()->preempt_count) += (1); } while (0); __asm__ __volatile__("": : :"memory"); } while (0); (void)0; } static inline __attribute__((always_inline)) void rcu_read_unlock_sched(void) { lock_release(&rcu_sched_lock_map, 1, ({ __label__ __here; __here: (unsigned long)&&__here; })); (void)0; do { do { __asm__ __volatile__("": : :"memory"); sub_preempt_count(1); } while (0); __asm__ __volatile__("": : :"memory"); do { if (__builtin_constant_p((((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/rcupdate.h", .line = 681, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/rcupdate.h", .line = 681, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/rcupdate.h", .line = 681, }; ______r = !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/rcupdate.h", .line = 681, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) preempt_schedule(); } while (0); } while (0); } static inline __attribute__((always_inline)) __attribute__((no_instrument_function)) void rcu_read_unlock_sched_notrace(void) { (void)0; do { do { __asm__ __volatile__("": : :"memory"); do { (current_thread_info()->preempt_count) -= (1); } while (0); } while (0); __asm__ __volatile__("": : :"memory"); do { if (__builtin_constant_p((((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/rcupdate.h", .line = 688, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/rcupdate.h", .line = 688, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/rcupdate.h", .line = 688, }; ______r = !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/rcupdate.h", .line = 688, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) preempt_schedule(); } while (0); } while (0); } struct rcu_synchronize { struct rcu_head head; struct completion completion; }; extern void wakeme_after_rcu(struct rcu_head *head); extern void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *head)); extern void call_rcu_bh(struct rcu_head *head, void (*func)(struct rcu_head *head)); static inline __attribute__((always_inline)) void debug_rcu_head_queue(struct rcu_head *head) { } static inline __attribute__((always_inline)) void debug_rcu_head_unqueue(struct rcu_head *head) { } static inline __attribute__((always_inline)) __attribute__((always_inline)) bool __is_kfree_rcu_offset(unsigned long offset) { return offset < 4096; } static inline __attribute__((always_inline)) __attribute__((always_inline)) void __kfree_rcu(struct rcu_head *head, unsigned long offset) { typedef void (*rcu_callback)(struct rcu_head *); do { ((void)sizeof(char[1 - 2*!!(!__builtin_constant_p(offset))])); if (__builtin_constant_p(((!__builtin_constant_p(offset)))) ? !!((!__builtin_constant_p(offset))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/rcupdate.h", .line = 822, }; ______r = !!((!__builtin_constant_p(offset))); ______f.miss_hit[______r]++; ______r; })) __build_bug_on_failed = 1; } while(0); do { ((void)sizeof(char[1 - 2*!!(!__is_kfree_rcu_offset(offset))])); if (__builtin_constant_p(((!__is_kfree_rcu_offset(offset)))) ? !!((!__is_kfree_rcu_offset(offset))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/rcupdate.h", .line = 825, }; ______r = !!((!__is_kfree_rcu_offset(offset))); ______f.miss_hit[______r]++; ______r; })) __build_bug_on_failed = 1; } while(0); call_rcu(head, (rcu_callback)offset); } extern void kfree(const void *); static inline __attribute__((always_inline)) void __rcu_reclaim(struct rcu_head *head) { unsigned long offset = (unsigned long)head->func; if (__builtin_constant_p(((__is_kfree_rcu_offset(offset)))) ? !!((__is_kfree_rcu_offset(offset))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/rcupdate.h", .line = 836, }; ______r = !!((__is_kfree_rcu_offset(offset))); ______f.miss_hit[______r]++; ______r; })) kfree((void *)head - offset); else head->func(head); } struct ctl_table; struct nsproxy; struct ctl_table_root; struct ctl_table_set { struct list_head list; struct ctl_table_set *parent; int (*is_seen)(struct ctl_table_set *); }; extern void setup_sysctl_set(struct ctl_table_set *p, struct ctl_table_set *parent, int (*is_seen)(struct ctl_table_set *)); struct ctl_table_header; extern void sysctl_head_get(struct ctl_table_header *); extern void sysctl_head_put(struct ctl_table_header *); extern int sysctl_is_seen(struct ctl_table_header *); extern struct ctl_table_header *sysctl_head_grab(struct ctl_table_header *); extern struct ctl_table_header *sysctl_head_next(struct ctl_table_header *prev); extern struct ctl_table_header *__sysctl_head_next(struct nsproxy *namespaces, struct ctl_table_header *prev); extern void sysctl_head_finish(struct ctl_table_header *prev); extern int sysctl_perm(struct ctl_table_root *root, struct ctl_table *table, int op); typedef struct ctl_table ctl_table; typedef int proc_handler (struct ctl_table *ctl, int write, void *buffer, size_t *lenp, loff_t *ppos); extern int proc_dostring(struct ctl_table *, int, void *, size_t *, loff_t *); extern int proc_dointvec(struct ctl_table *, int, void *, size_t *, loff_t *); extern int proc_dointvec_minmax(struct ctl_table *, int, void *, size_t *, loff_t *); extern int proc_dointvec_jiffies(struct ctl_table *, int, void *, size_t *, loff_t *); extern int proc_dointvec_userhz_jiffies(struct ctl_table *, int, void *, size_t *, loff_t *); extern int proc_dointvec_ms_jiffies(struct ctl_table *, int, void *, size_t *, loff_t *); extern int proc_doulongvec_minmax(struct ctl_table *, int, void *, size_t *, loff_t *); extern int proc_doulongvec_ms_jiffies_minmax(struct ctl_table *table, int, void *, size_t *, loff_t *); extern int proc_do_large_bitmap(struct ctl_table *, int, void *, size_t *, loff_t *); struct ctl_table { const char *procname; void *data; int maxlen; mode_t mode; struct ctl_table *child; struct ctl_table *parent; proc_handler *proc_handler; void *extra1; void *extra2; }; struct ctl_table_root { struct list_head root_list; struct ctl_table_set default_set; struct ctl_table_set *(*lookup)(struct ctl_table_root *root, struct nsproxy *namespaces); int (*permissions)(struct ctl_table_root *root, struct nsproxy *namespaces, struct ctl_table *table); }; struct ctl_table_header { union { struct { struct ctl_table *ctl_table; struct list_head ctl_entry; int used; int count; }; struct rcu_head rcu; }; struct completion *unregistering; struct ctl_table *ctl_table_arg; struct ctl_table_root *root; struct ctl_table_set *set; struct ctl_table *attached_by; struct ctl_table *attached_to; struct ctl_table_header *parent; }; struct ctl_path { const char *procname; }; void register_sysctl_root(struct ctl_table_root *root); struct ctl_table_header *__register_sysctl_paths( struct ctl_table_root *root, struct nsproxy *namespaces, const struct ctl_path *path, struct ctl_table *table); struct ctl_table_header *register_sysctl_table(struct ctl_table * table); struct ctl_table_header *register_sysctl_paths(const struct ctl_path *path, struct ctl_table *table); void unregister_sysctl_table(struct ctl_table_header * table); int sysctl_check_table(struct nsproxy *namespaces, struct ctl_table *table); extern char modprobe_path[]; extern int __request_module(bool wait, const char *name, ...) __attribute__((format(printf, 2, 3))); struct cred; struct file; enum umh_wait { UMH_NO_WAIT = -1, UMH_WAIT_EXEC = 0, UMH_WAIT_PROC = 1, }; struct subprocess_info { struct work_struct work; struct completion *complete; char *path; char **argv; char **envp; enum umh_wait wait; int retval; int (*init)(struct subprocess_info *info, struct cred *new); void (*cleanup)(struct subprocess_info *info); void *data; }; struct subprocess_info *call_usermodehelper_setup(char *path, char **argv, char **envp, gfp_t gfp_mask); void call_usermodehelper_setfns(struct subprocess_info *info, int (*init)(struct subprocess_info *info, struct cred *new), void (*cleanup)(struct subprocess_info *info), void *data); int call_usermodehelper_exec(struct subprocess_info *info, enum umh_wait wait); void call_usermodehelper_freeinfo(struct subprocess_info *info); static inline __attribute__((always_inline)) int call_usermodehelper_fns(char *path, char **argv, char **envp, enum umh_wait wait, int (*init)(struct subprocess_info *info, struct cred *new), void (*cleanup)(struct subprocess_info *), void *data) { struct subprocess_info *info; gfp_t gfp_mask = (wait == UMH_NO_WAIT) ? ((( gfp_t)0x20u)) : ((( gfp_t)0x10u) | (( gfp_t)0x40u) | (( gfp_t)0x80u)); info = call_usermodehelper_setup(path, argv, envp, gfp_mask); if (__builtin_constant_p(((info == ((void *)0)))) ? !!((info == ((void *)0))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/kmod.h", .line = 98, }; ______r = !!((info == ((void *)0))); ______f.miss_hit[______r]++; ______r; })) return -12; call_usermodehelper_setfns(info, init, cleanup, data); return call_usermodehelper_exec(info, wait); } static inline __attribute__((always_inline)) int call_usermodehelper(char *path, char **argv, char **envp, enum umh_wait wait) { return call_usermodehelper_fns(path, argv, envp, wait, ((void *)0), ((void *)0), ((void *)0)); } extern struct ctl_table usermodehelper_table[]; extern void usermodehelper_init(void); extern int usermodehelper_disable(void); extern void usermodehelper_enable(void); extern bool usermodehelper_is_disabled(void); struct user_i387_struct { long cwd; long swd; long twd; long fip; long fcs; long foo; long fos; long st_space[20]; }; struct user_fxsr_struct { unsigned short cwd; unsigned short swd; unsigned short twd; unsigned short fop; long fip; long fcs; long foo; long fos; long mxcsr; long reserved; long st_space[32]; long xmm_space[32]; long padding[56]; }; struct user_regs_struct { unsigned long bx; unsigned long cx; unsigned long dx; unsigned long si; unsigned long di; unsigned long bp; unsigned long ax; unsigned long ds; unsigned long es; unsigned long fs; unsigned long gs; unsigned long orig_ax; unsigned long ip; unsigned long cs; unsigned long flags; unsigned long sp; unsigned long ss; }; struct user{ struct user_regs_struct regs; int u_fpvalid; struct user_i387_struct i387; unsigned long int u_tsize; unsigned long int u_dsize; unsigned long int u_ssize; unsigned long start_code; unsigned long start_stack; long int signal; int reserved; unsigned long u_ar0; struct user_i387_struct *u_fpstate; unsigned long magic; char u_comm[32]; int u_debugreg[8]; }; struct user_ymmh_regs { __u32 ymmh_space[64]; }; struct user_xsave_hdr { __u64 xstate_bv; __u64 reserved1[2]; __u64 reserved2[5]; }; struct user_xstateregs { struct { __u64 fpx_space[58]; __u64 xstate_fx_sw[6]; } i387; struct user_xsave_hdr xsave_hdr; struct user_ymmh_regs ymmh; }; typedef unsigned long elf_greg_t; typedef elf_greg_t elf_gregset_t[(sizeof(struct user_regs_struct) / sizeof(elf_greg_t))]; typedef struct user_i387_struct elf_fpregset_t; typedef struct user_fxsr_struct elf_fpxregset_t; extern const char VDSO32_PRELINK[]; extern void __kernel_sigreturn; extern void __kernel_rt_sigreturn; extern const char vdso32_int80_start, vdso32_int80_end; extern const char vdso32_syscall_start, vdso32_syscall_end; extern const char vdso32_sysenter_start, vdso32_sysenter_end; extern unsigned int vdso_enabled; struct user_desc { unsigned int entry_number; unsigned int base_addr; unsigned int limit; unsigned int seg_32bit:1; unsigned int contents:2; unsigned int read_exec_only:1; unsigned int limit_in_pages:1; unsigned int seg_not_present:1; unsigned int useable:1; }; static inline __attribute__((always_inline)) void fill_ldt(struct desc_struct *desc, const struct user_desc *info) { desc->limit0 = info->limit & 0x0ffff; desc->base0 = (info->base_addr & 0x0000ffff); desc->base1 = (info->base_addr & 0x00ff0000) >> 16; desc->type = (info->read_exec_only ^ 1) << 1; desc->type |= info->contents << 2; desc->s = 1; desc->dpl = 0x3; desc->p = info->seg_not_present ^ 1; desc->limit = (info->limit & 0xf0000) >> 16; desc->avl = info->useable; desc->d = info->seg_32bit; desc->g = info->limit_in_pages; desc->base2 = (info->base_addr & 0xff000000) >> 24; desc->l = 0; } extern struct desc_ptr idt_descr; extern gate_desc idt_table[]; struct gdt_page { struct desc_struct gdt[32]; } __attribute__((aligned(((1UL) << 12)))); extern __attribute__((section(".data..percpu" "..page_aligned"))) __typeof__(struct gdt_page) gdt_page __attribute__((aligned(((1UL) << 12)))); static inline __attribute__((always_inline)) struct desc_struct *get_cpu_gdt_table(unsigned int cpu) { return (*({ do { const void *__vpp_verify = (typeof((&(gdt_page))))((void *)0); (void)__vpp_verify; } while (0); ({ unsigned long __ptr; __asm__ ("" : "=r"(__ptr) : "0"((typeof(*(&(gdt_page))) *)(&(gdt_page)))); (typeof((typeof(*(&(gdt_page))) *)(&(gdt_page)))) (__ptr + (((__per_cpu_offset[cpu])))); }); })).gdt; } static inline __attribute__((always_inline)) void pack_gate(gate_desc *gate, unsigned char type, unsigned long base, unsigned dpl, unsigned flags, unsigned short seg) { gate->a = (seg << 16) | (base & 0xffff); gate->b = (base & 0xffff0000) | (((0x80 | type | (dpl << 5)) & 0xff) << 8); } static inline __attribute__((always_inline)) int desc_empty(const void *ptr) { const u32 *desc = ptr; return !(desc[0] | desc[1]); } static inline __attribute__((always_inline)) void native_write_idt_entry(gate_desc *idt, int entry, const gate_desc *gate) { __builtin_memcpy(&idt[entry], gate, sizeof(*gate)); } static inline __attribute__((always_inline)) void native_write_ldt_entry(struct desc_struct *ldt, int entry, const void *desc) { __builtin_memcpy(&ldt[entry], desc, 8); } static inline __attribute__((always_inline)) void native_write_gdt_entry(struct desc_struct *gdt, int entry, const void *desc, int type) { unsigned int size; switch (type) { case DESC_TSS: size = sizeof(tss_desc); break; case DESC_LDT: size = sizeof(ldt_desc); break; default: size = sizeof(*gdt); break; } __builtin_memcpy(&gdt[entry], desc, size); } static inline __attribute__((always_inline)) void pack_descriptor(struct desc_struct *desc, unsigned long base, unsigned long limit, unsigned char type, unsigned char flags) { desc->a = ((base & 0xffff) << 16) | (limit & 0xffff); desc->b = (base & 0xff000000) | ((base & 0xff0000) >> 16) | (limit & 0x000f0000) | ((type & 0xff) << 8) | ((flags & 0xf) << 20); desc->p = 1; } static inline __attribute__((always_inline)) void set_tssldt_descriptor(void *d, unsigned long addr, unsigned type, unsigned size) { pack_descriptor((struct desc_struct *)d, addr, size, 0x80 | type, 0); } static inline __attribute__((always_inline)) void __set_tss_desc(unsigned cpu, unsigned int entry, void *addr) { struct desc_struct *d = get_cpu_gdt_table(cpu); tss_desc tss; set_tssldt_descriptor(&tss, (unsigned long)addr, DESC_TSS, __builtin_offsetof(struct tss_struct,io_bitmap) + (65536/8) + sizeof(unsigned long) - 1); write_gdt_entry(d, entry, &tss, DESC_TSS); } static inline __attribute__((always_inline)) void native_set_ldt(const void *addr, unsigned int entries) { if (__builtin_constant_p((((__builtin_constant_p(entries == 0) ? !!(entries == 0) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/desc.h", .line = 194, }; ______r = __builtin_expect(!!(entries == 0), 1); ftrace_likely_update(&______f, ______r, 1); ______r; }))))) ? !!(((__builtin_constant_p(entries == 0) ? !!(entries == 0) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/desc.h", .line = 194, }; ______r = __builtin_expect(!!(entries == 0), 1); ftrace_likely_update(&______f, ______r, 1); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/desc.h", .line = 194, }; ______r = !!(((__builtin_constant_p(entries == 0) ? !!(entries == 0) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/desc.h", .line = 194, }; ______r = __builtin_expect(!!(entries == 0), 1); ftrace_likely_update(&______f, ______r, 1); ______r; })))); ______f.miss_hit[______r]++; ______r; })) asm volatile("lldt %w0"::"q" (0)); else { unsigned cpu = debug_smp_processor_id(); ldt_desc ldt; set_tssldt_descriptor(&ldt, (unsigned long)addr, DESC_LDT, entries * 8 - 1); write_gdt_entry(get_cpu_gdt_table(cpu), ((12)+5), &ldt, DESC_LDT); asm volatile("lldt %w0"::"q" (((12)+5)*8)); } } static inline __attribute__((always_inline)) void native_load_tr_desc(void) { asm volatile("ltr %w0"::"q" (((12)+4)*8)); } static inline __attribute__((always_inline)) void native_load_gdt(const struct desc_ptr *dtr) { asm volatile("lgdt %0"::"m" (*dtr)); } static inline __attribute__((always_inline)) void native_load_idt(const struct desc_ptr *dtr) { asm volatile("lidt %0"::"m" (*dtr)); } static inline __attribute__((always_inline)) void native_store_gdt(struct desc_ptr *dtr) { asm volatile("sgdt %0":"=m" (*dtr)); } static inline __attribute__((always_inline)) void native_store_idt(struct desc_ptr *dtr) { asm volatile("sidt %0":"=m" (*dtr)); } static inline __attribute__((always_inline)) unsigned long native_store_tr(void) { unsigned long tr; asm volatile("str %0":"=r" (tr)); return tr; } static inline __attribute__((always_inline)) void native_load_tls(struct thread_struct *t, unsigned int cpu) { struct desc_struct *gdt = get_cpu_gdt_table(cpu); unsigned int i; for (i = 0; i < 3; i++) gdt[6 + i] = t->tls_array[i]; } static inline __attribute__((always_inline)) void clear_LDT(void) { set_ldt(((void *)0), 0); } static inline __attribute__((always_inline)) void load_LDT_nolock(mm_context_t *pc) { set_ldt(pc->ldt, pc->size); } static inline __attribute__((always_inline)) void load_LDT(mm_context_t *pc) { do { add_preempt_count(1); __asm__ __volatile__("": : :"memory"); } while (0); load_LDT_nolock(pc); do { do { __asm__ __volatile__("": : :"memory"); sub_preempt_count(1); } while (0); __asm__ __volatile__("": : :"memory"); do { if (__builtin_constant_p((((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/desc.h", .line = 284, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/desc.h", .line = 284, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/desc.h", .line = 284, }; ______r = !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/desc.h", .line = 284, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) preempt_schedule(); } while (0); } while (0); } static inline __attribute__((always_inline)) unsigned long get_desc_base(const struct desc_struct *desc) { return (unsigned)(desc->base0 | ((desc->base1) << 16) | ((desc->base2) << 24)); } static inline __attribute__((always_inline)) void set_desc_base(struct desc_struct *desc, unsigned long base) { desc->base0 = base & 0xffff; desc->base1 = (base >> 16) & 0xff; desc->base2 = (base >> 24) & 0xff; } static inline __attribute__((always_inline)) unsigned long get_desc_limit(const struct desc_struct *desc) { return desc->limit0 | (desc->limit << 16); } static inline __attribute__((always_inline)) void set_desc_limit(struct desc_struct *desc, unsigned long limit) { desc->limit0 = limit & 0xffff; desc->limit = (limit >> 16) & 0xf; } static inline __attribute__((always_inline)) void _set_gate(int gate, unsigned type, void *addr, unsigned dpl, unsigned ist, unsigned seg) { gate_desc s; pack_gate(&s, type, (unsigned long)addr, dpl, ist, seg); write_idt_entry(idt_table, gate, &s); } static inline __attribute__((always_inline)) void set_intr_gate(unsigned int n, void *addr) { do { if (__builtin_constant_p((((__builtin_constant_p((unsigned)n > 0xFF) ? !!((unsigned)n > 0xFF) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/desc.h", .line = 331, }; ______r = __builtin_expect(!!((unsigned)n > 0xFF), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p((unsigned)n > 0xFF) ? !!((unsigned)n > 0xFF) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/desc.h", .line = 331, }; ______r = __builtin_expect(!!((unsigned)n > 0xFF), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/desc.h", .line = 331, }; ______r = !!(((__builtin_constant_p((unsigned)n > 0xFF) ? !!((unsigned)n > 0xFF) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/desc.h", .line = 331, }; ______r = __builtin_expect(!!((unsigned)n > 0xFF), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) do { asm volatile("1:\tud2\n" ".pushsection __bug_table,\"a\"\n" "2:\t.long 1b, %c0\n" "\t.word %c1, 0\n" "\t.org 2b+%c2\n" ".popsection" : : "i" ("/data/exp/linux-3.0.4/arch/x86/include/asm/desc.h"), "i" (331), "i" (sizeof(struct bug_entry))); __builtin_unreachable(); } while (0); } while(0); _set_gate(n, GATE_INTERRUPT, addr, 0, 0, (((12)+0)*8)); } extern int first_system_vector; extern unsigned long used_vectors[]; static inline __attribute__((always_inline)) void alloc_system_vector(int vector) { if (__builtin_constant_p(((!(__builtin_constant_p((vector)) ? constant_test_bit((vector), (used_vectors)) : variable_test_bit((vector), (used_vectors)))))) ? !!((!(__builtin_constant_p((vector)) ? constant_test_bit((vector), (used_vectors)) : variable_test_bit((vector), (used_vectors))))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/desc.h", .line = 341, }; ______r = !!((!(__builtin_constant_p((vector)) ? constant_test_bit((vector), (used_vectors)) : variable_test_bit((vector), (used_vectors))))); ______f.miss_hit[______r]++; ______r; })) { set_bit(vector, used_vectors); if (__builtin_constant_p(((first_system_vector > vector))) ? !!((first_system_vector > vector)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/desc.h", .line = 343, }; ______r = !!((first_system_vector > vector)); ______f.miss_hit[______r]++; ______r; })) first_system_vector = vector; } else { do { asm volatile("1:\tud2\n" ".pushsection __bug_table,\"a\"\n" "2:\t.long 1b, %c0\n" "\t.word %c1, 0\n" "\t.org 2b+%c2\n" ".popsection" : : "i" ("/data/exp/linux-3.0.4/arch/x86/include/asm/desc.h"), "i" (346), "i" (sizeof(struct bug_entry))); __builtin_unreachable(); } while (0); } } static inline __attribute__((always_inline)) void alloc_intr_gate(unsigned int n, void *addr) { alloc_system_vector(n); set_intr_gate(n, addr); } static inline __attribute__((always_inline)) void set_system_intr_gate(unsigned int n, void *addr) { do { if (__builtin_constant_p((((__builtin_constant_p((unsigned)n > 0xFF) ? !!((unsigned)n > 0xFF) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/desc.h", .line = 361, }; ______r = __builtin_expect(!!((unsigned)n > 0xFF), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p((unsigned)n > 0xFF) ? !!((unsigned)n > 0xFF) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/desc.h", .line = 361, }; ______r = __builtin_expect(!!((unsigned)n > 0xFF), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/desc.h", .line = 361, }; ______r = !!(((__builtin_constant_p((unsigned)n > 0xFF) ? !!((unsigned)n > 0xFF) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/desc.h", .line = 361, }; ______r = __builtin_expect(!!((unsigned)n > 0xFF), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) do { asm volatile("1:\tud2\n" ".pushsection __bug_table,\"a\"\n" "2:\t.long 1b, %c0\n" "\t.word %c1, 0\n" "\t.org 2b+%c2\n" ".popsection" : : "i" ("/data/exp/linux-3.0.4/arch/x86/include/asm/desc.h"), "i" (361), "i" (sizeof(struct bug_entry))); __builtin_unreachable(); } while (0); } while(0); _set_gate(n, GATE_INTERRUPT, addr, 0x3, 0, (((12)+0)*8)); } static inline __attribute__((always_inline)) void set_system_trap_gate(unsigned int n, void *addr) { do { if (__builtin_constant_p((((__builtin_constant_p((unsigned)n > 0xFF) ? !!((unsigned)n > 0xFF) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/desc.h", .line = 367, }; ______r = __builtin_expect(!!((unsigned)n > 0xFF), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p((unsigned)n > 0xFF) ? !!((unsigned)n > 0xFF) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/desc.h", .line = 367, }; ______r = __builtin_expect(!!((unsigned)n > 0xFF), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/desc.h", .line = 367, }; ______r = !!(((__builtin_constant_p((unsigned)n > 0xFF) ? !!((unsigned)n > 0xFF) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/desc.h", .line = 367, }; ______r = __builtin_expect(!!((unsigned)n > 0xFF), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) do { asm volatile("1:\tud2\n" ".pushsection __bug_table,\"a\"\n" "2:\t.long 1b, %c0\n" "\t.word %c1, 0\n" "\t.org 2b+%c2\n" ".popsection" : : "i" ("/data/exp/linux-3.0.4/arch/x86/include/asm/desc.h"), "i" (367), "i" (sizeof(struct bug_entry))); __builtin_unreachable(); } while (0); } while(0); _set_gate(n, GATE_TRAP, addr, 0x3, 0, (((12)+0)*8)); } static inline __attribute__((always_inline)) void set_trap_gate(unsigned int n, void *addr) { do { if (__builtin_constant_p((((__builtin_constant_p((unsigned)n > 0xFF) ? !!((unsigned)n > 0xFF) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/desc.h", .line = 373, }; ______r = __builtin_expect(!!((unsigned)n > 0xFF), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p((unsigned)n > 0xFF) ? !!((unsigned)n > 0xFF) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/desc.h", .line = 373, }; ______r = __builtin_expect(!!((unsigned)n > 0xFF), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/desc.h", .line = 373, }; ______r = !!(((__builtin_constant_p((unsigned)n > 0xFF) ? !!((unsigned)n > 0xFF) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/desc.h", .line = 373, }; ______r = __builtin_expect(!!((unsigned)n > 0xFF), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) do { asm volatile("1:\tud2\n" ".pushsection __bug_table,\"a\"\n" "2:\t.long 1b, %c0\n" "\t.word %c1, 0\n" "\t.org 2b+%c2\n" ".popsection" : : "i" ("/data/exp/linux-3.0.4/arch/x86/include/asm/desc.h"), "i" (373), "i" (sizeof(struct bug_entry))); __builtin_unreachable(); } while (0); } while(0); _set_gate(n, GATE_TRAP, addr, 0, 0, (((12)+0)*8)); } static inline __attribute__((always_inline)) void set_task_gate(unsigned int n, unsigned int gdt_entry) { do { if (__builtin_constant_p((((__builtin_constant_p((unsigned)n > 0xFF) ? !!((unsigned)n > 0xFF) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/desc.h", .line = 379, }; ______r = __builtin_expect(!!((unsigned)n > 0xFF), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p((unsigned)n > 0xFF) ? !!((unsigned)n > 0xFF) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/desc.h", .line = 379, }; ______r = __builtin_expect(!!((unsigned)n > 0xFF), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/desc.h", .line = 379, }; ______r = !!(((__builtin_constant_p((unsigned)n > 0xFF) ? !!((unsigned)n > 0xFF) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/desc.h", .line = 379, }; ______r = __builtin_expect(!!((unsigned)n > 0xFF), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) do { asm volatile("1:\tud2\n" ".pushsection __bug_table,\"a\"\n" "2:\t.long 1b, %c0\n" "\t.word %c1, 0\n" "\t.org 2b+%c2\n" ".popsection" : : "i" ("/data/exp/linux-3.0.4/arch/x86/include/asm/desc.h"), "i" (379), "i" (sizeof(struct bug_entry))); __builtin_unreachable(); } while (0); } while(0); _set_gate(n, GATE_TASK, (void *)0, 0, 0, (gdt_entry<<3)); } static inline __attribute__((always_inline)) void set_intr_gate_ist(int n, void *addr, unsigned ist) { do { if (__builtin_constant_p((((__builtin_constant_p((unsigned)n > 0xFF) ? !!((unsigned)n > 0xFF) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/desc.h", .line = 385, }; ______r = __builtin_expect(!!((unsigned)n > 0xFF), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p((unsigned)n > 0xFF) ? !!((unsigned)n > 0xFF) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/desc.h", .line = 385, }; ______r = __builtin_expect(!!((unsigned)n > 0xFF), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/desc.h", .line = 385, }; ______r = !!(((__builtin_constant_p((unsigned)n > 0xFF) ? !!((unsigned)n > 0xFF) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/desc.h", .line = 385, }; ______r = __builtin_expect(!!((unsigned)n > 0xFF), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) do { asm volatile("1:\tud2\n" ".pushsection __bug_table,\"a\"\n" "2:\t.long 1b, %c0\n" "\t.word %c1, 0\n" "\t.org 2b+%c2\n" ".popsection" : : "i" ("/data/exp/linux-3.0.4/arch/x86/include/asm/desc.h"), "i" (385), "i" (sizeof(struct bug_entry))); __builtin_unreachable(); } while (0); } while(0); _set_gate(n, GATE_INTERRUPT, addr, 0, ist, (((12)+0)*8)); } static inline __attribute__((always_inline)) void set_system_intr_gate_ist(int n, void *addr, unsigned ist) { do { if (__builtin_constant_p((((__builtin_constant_p((unsigned)n > 0xFF) ? !!((unsigned)n > 0xFF) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/desc.h", .line = 391, }; ______r = __builtin_expect(!!((unsigned)n > 0xFF), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p((unsigned)n > 0xFF) ? !!((unsigned)n > 0xFF) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/desc.h", .line = 391, }; ______r = __builtin_expect(!!((unsigned)n > 0xFF), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/desc.h", .line = 391, }; ______r = !!(((__builtin_constant_p((unsigned)n > 0xFF) ? !!((unsigned)n > 0xFF) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/desc.h", .line = 391, }; ______r = __builtin_expect(!!((unsigned)n > 0xFF), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) do { asm volatile("1:\tud2\n" ".pushsection __bug_table,\"a\"\n" "2:\t.long 1b, %c0\n" "\t.word %c1, 0\n" "\t.org 2b+%c2\n" ".popsection" : : "i" ("/data/exp/linux-3.0.4/arch/x86/include/asm/desc.h"), "i" (391), "i" (sizeof(struct bug_entry))); __builtin_unreachable(); } while (0); } while(0); _set_gate(n, GATE_INTERRUPT, addr, 0x3, ist, (((12)+0)*8)); } struct task_struct; struct linux_binprm; extern int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp); extern int syscall32_setup_pages(struct linux_binprm *, int exstack); extern unsigned long arch_randomize_brk(struct mm_struct *mm); struct file; typedef __u32 Elf32_Addr; typedef __u16 Elf32_Half; typedef __u32 Elf32_Off; typedef __s32 Elf32_Sword; typedef __u32 Elf32_Word; typedef __u64 Elf64_Addr; typedef __u16 Elf64_Half; typedef __s16 Elf64_SHalf; typedef __u64 Elf64_Off; typedef __s32 Elf64_Sword; typedef __u32 Elf64_Word; typedef __u64 Elf64_Xword; typedef __s64 Elf64_Sxword; typedef struct dynamic{ Elf32_Sword d_tag; union{ Elf32_Sword d_val; Elf32_Addr d_ptr; } d_un; } Elf32_Dyn; typedef struct { Elf64_Sxword d_tag; union { Elf64_Xword d_val; Elf64_Addr d_ptr; } d_un; } Elf64_Dyn; typedef struct elf32_rel { Elf32_Addr r_offset; Elf32_Word r_info; } Elf32_Rel; typedef struct elf64_rel { Elf64_Addr r_offset; Elf64_Xword r_info; } Elf64_Rel; typedef struct elf32_rela{ Elf32_Addr r_offset; Elf32_Word r_info; Elf32_Sword r_addend; } Elf32_Rela; typedef struct elf64_rela { Elf64_Addr r_offset; Elf64_Xword r_info; Elf64_Sxword r_addend; } Elf64_Rela; typedef struct elf32_sym{ Elf32_Word st_name; Elf32_Addr st_value; Elf32_Word st_size; unsigned char st_info; unsigned char st_other; Elf32_Half st_shndx; } Elf32_Sym; typedef struct elf64_sym { Elf64_Word st_name; unsigned char st_info; unsigned char st_other; Elf64_Half st_shndx; Elf64_Addr st_value; Elf64_Xword st_size; } Elf64_Sym; typedef struct elf32_hdr{ unsigned char e_ident[16]; Elf32_Half e_type; Elf32_Half e_machine; Elf32_Word e_version; Elf32_Addr e_entry; Elf32_Off e_phoff; Elf32_Off e_shoff; Elf32_Word e_flags; Elf32_Half e_ehsize; Elf32_Half e_phentsize; Elf32_Half e_phnum; Elf32_Half e_shentsize; Elf32_Half e_shnum; Elf32_Half e_shstrndx; } Elf32_Ehdr; typedef struct elf64_hdr { unsigned char e_ident[16]; Elf64_Half e_type; Elf64_Half e_machine; Elf64_Word e_version; Elf64_Addr e_entry; Elf64_Off e_phoff; Elf64_Off e_shoff; Elf64_Word e_flags; Elf64_Half e_ehsize; Elf64_Half e_phentsize; Elf64_Half e_phnum; Elf64_Half e_shentsize; Elf64_Half e_shnum; Elf64_Half e_shstrndx; } Elf64_Ehdr; typedef struct elf32_phdr{ Elf32_Word p_type; Elf32_Off p_offset; Elf32_Addr p_vaddr; Elf32_Addr p_paddr; Elf32_Word p_filesz; Elf32_Word p_memsz; Elf32_Word p_flags; Elf32_Word p_align; } Elf32_Phdr; typedef struct elf64_phdr { Elf64_Word p_type; Elf64_Word p_flags; Elf64_Off p_offset; Elf64_Addr p_vaddr; Elf64_Addr p_paddr; Elf64_Xword p_filesz; Elf64_Xword p_memsz; Elf64_Xword p_align; } Elf64_Phdr; typedef struct elf32_shdr { Elf32_Word sh_name; Elf32_Word sh_type; Elf32_Word sh_flags; Elf32_Addr sh_addr; Elf32_Off sh_offset; Elf32_Word sh_size; Elf32_Word sh_link; Elf32_Word sh_info; Elf32_Word sh_addralign; Elf32_Word sh_entsize; } Elf32_Shdr; typedef struct elf64_shdr { Elf64_Word sh_name; Elf64_Word sh_type; Elf64_Xword sh_flags; Elf64_Addr sh_addr; Elf64_Off sh_offset; Elf64_Xword sh_size; Elf64_Word sh_link; Elf64_Word sh_info; Elf64_Xword sh_addralign; Elf64_Xword sh_entsize; } Elf64_Shdr; typedef struct elf32_note { Elf32_Word n_namesz; Elf32_Word n_descsz; Elf32_Word n_type; } Elf32_Nhdr; typedef struct elf64_note { Elf64_Word n_namesz; Elf64_Word n_descsz; Elf64_Word n_type; } Elf64_Nhdr; extern Elf32_Dyn _DYNAMIC []; static inline __attribute__((always_inline)) int elf_coredump_extra_notes_size(void) { return 0; } static inline __attribute__((always_inline)) int elf_coredump_extra_notes_write(struct file *file, loff_t *foffset) { return 0; } struct sock; struct kobject; enum kobj_ns_type { KOBJ_NS_TYPE_NONE = 0, KOBJ_NS_TYPE_NET, KOBJ_NS_TYPES }; struct kobj_ns_type_operations { enum kobj_ns_type type; void *(*grab_current_ns)(void); const void *(*netlink_ns)(struct sock *sk); const void *(*initial_ns)(void); void (*drop_ns)(void *); }; int kobj_ns_type_register(const struct kobj_ns_type_operations *ops); int kobj_ns_type_registered(enum kobj_ns_type type); const struct kobj_ns_type_operations *kobj_child_ns_ops(struct kobject *parent); const struct kobj_ns_type_operations *kobj_ns_ops(struct kobject *kobj); void *kobj_ns_grab_current(enum kobj_ns_type type); const void *kobj_ns_netlink(enum kobj_ns_type type, struct sock *sk); const void *kobj_ns_initial(enum kobj_ns_type type); void kobj_ns_drop(enum kobj_ns_type type, void *ns); struct kobject; struct module; enum kobj_ns_type; struct attribute { const char *name; mode_t mode; struct lock_class_key *key; struct lock_class_key skey; }; struct attribute_group { const char *name; mode_t (*is_visible)(struct kobject *, struct attribute *, int); struct attribute **attrs; }; struct file; struct vm_area_struct; struct bin_attribute { struct attribute attr; size_t size; void *private; ssize_t (*read)(struct file *, struct kobject *, struct bin_attribute *, char *, loff_t, size_t); ssize_t (*write)(struct file *,struct kobject *, struct bin_attribute *, char *, loff_t, size_t); int (*mmap)(struct file *, struct kobject *, struct bin_attribute *attr, struct vm_area_struct *vma); }; struct sysfs_ops { ssize_t (*show)(struct kobject *, struct attribute *,char *); ssize_t (*store)(struct kobject *,struct attribute *,const char *, size_t); }; struct sysfs_dirent; int sysfs_schedule_callback(struct kobject *kobj, void (*func)(void *), void *data, struct module *owner); int __attribute__((warn_unused_result)) sysfs_create_dir(struct kobject *kobj); void sysfs_remove_dir(struct kobject *kobj); int __attribute__((warn_unused_result)) sysfs_rename_dir(struct kobject *kobj, const char *new_name); int __attribute__((warn_unused_result)) sysfs_move_dir(struct kobject *kobj, struct kobject *new_parent_kobj); int __attribute__((warn_unused_result)) sysfs_create_file(struct kobject *kobj, const struct attribute *attr); int __attribute__((warn_unused_result)) sysfs_create_files(struct kobject *kobj, const struct attribute **attr); int __attribute__((warn_unused_result)) sysfs_chmod_file(struct kobject *kobj, const struct attribute *attr, mode_t mode); void sysfs_remove_file(struct kobject *kobj, const struct attribute *attr); void sysfs_remove_files(struct kobject *kobj, const struct attribute **attr); int __attribute__((warn_unused_result)) sysfs_create_bin_file(struct kobject *kobj, const struct bin_attribute *attr); void sysfs_remove_bin_file(struct kobject *kobj, const struct bin_attribute *attr); int __attribute__((warn_unused_result)) sysfs_create_link(struct kobject *kobj, struct kobject *target, const char *name); int __attribute__((warn_unused_result)) sysfs_create_link_nowarn(struct kobject *kobj, struct kobject *target, const char *name); void sysfs_remove_link(struct kobject *kobj, const char *name); int sysfs_rename_link(struct kobject *kobj, struct kobject *target, const char *old_name, const char *new_name); void sysfs_delete_link(struct kobject *dir, struct kobject *targ, const char *name); int __attribute__((warn_unused_result)) sysfs_create_group(struct kobject *kobj, const struct attribute_group *grp); int sysfs_update_group(struct kobject *kobj, const struct attribute_group *grp); void sysfs_remove_group(struct kobject *kobj, const struct attribute_group *grp); int sysfs_add_file_to_group(struct kobject *kobj, const struct attribute *attr, const char *group); void sysfs_remove_file_from_group(struct kobject *kobj, const struct attribute *attr, const char *group); int sysfs_merge_group(struct kobject *kobj, const struct attribute_group *grp); void sysfs_unmerge_group(struct kobject *kobj, const struct attribute_group *grp); void sysfs_notify(struct kobject *kobj, const char *dir, const char *attr); void sysfs_notify_dirent(struct sysfs_dirent *sd); struct sysfs_dirent *sysfs_get_dirent(struct sysfs_dirent *parent_sd, const void *ns, const unsigned char *name); struct sysfs_dirent *sysfs_get(struct sysfs_dirent *sd); void sysfs_put(struct sysfs_dirent *sd); int __attribute__((warn_unused_result)) sysfs_init(void); struct kref { atomic_t refcount; }; void kref_init(struct kref *kref); void kref_get(struct kref *kref); int kref_put(struct kref *kref, void (*release) (struct kref *kref)); int kref_sub(struct kref *kref, unsigned int count, void (*release) (struct kref *kref)); extern char uevent_helper[]; extern u64 uevent_seqnum; enum kobject_action { KOBJ_ADD, KOBJ_REMOVE, KOBJ_CHANGE, KOBJ_MOVE, KOBJ_ONLINE, KOBJ_OFFLINE, KOBJ_MAX }; struct kobject { const char *name; struct list_head entry; struct kobject *parent; struct kset *kset; struct kobj_type *ktype; struct sysfs_dirent *sd; struct kref kref; unsigned int state_initialized:1; unsigned int state_in_sysfs:1; unsigned int state_add_uevent_sent:1; unsigned int state_remove_uevent_sent:1; unsigned int uevent_suppress:1; }; extern int kobject_set_name(struct kobject *kobj, const char *name, ...) __attribute__((format(printf, 2, 3))); extern int kobject_set_name_vargs(struct kobject *kobj, const char *fmt, va_list vargs); static inline __attribute__((always_inline)) const char *kobject_name(const struct kobject *kobj) { return kobj->name; } extern void kobject_init(struct kobject *kobj, struct kobj_type *ktype); extern int __attribute__((warn_unused_result)) kobject_add(struct kobject *kobj, struct kobject *parent, const char *fmt, ...) __attribute__((format(printf, 3, 4))); extern int __attribute__((warn_unused_result)) kobject_init_and_add(struct kobject *kobj, struct kobj_type *ktype, struct kobject *parent, const char *fmt, ...) __attribute__((format(printf, 4, 5))); extern void kobject_del(struct kobject *kobj); extern struct kobject * __attribute__((warn_unused_result)) kobject_create(void); extern struct kobject * __attribute__((warn_unused_result)) kobject_create_and_add(const char *name, struct kobject *parent); extern int __attribute__((warn_unused_result)) kobject_rename(struct kobject *, const char *new_name); extern int __attribute__((warn_unused_result)) kobject_move(struct kobject *, struct kobject *); extern struct kobject *kobject_get(struct kobject *kobj); extern void kobject_put(struct kobject *kobj); extern char *kobject_get_path(struct kobject *kobj, gfp_t flag); struct kobj_type { void (*release)(struct kobject *kobj); const struct sysfs_ops *sysfs_ops; struct attribute **default_attrs; const struct kobj_ns_type_operations *(*child_ns_type)(struct kobject *kobj); const void *(*namespace)(struct kobject *kobj); }; struct kobj_uevent_env { char *envp[32]; int envp_idx; char buf[2048]; int buflen; }; struct kset_uevent_ops { int (* const filter)(struct kset *kset, struct kobject *kobj); const char *(* const name)(struct kset *kset, struct kobject *kobj); int (* const uevent)(struct kset *kset, struct kobject *kobj, struct kobj_uevent_env *env); }; struct kobj_attribute { struct attribute attr; ssize_t (*show)(struct kobject *kobj, struct kobj_attribute *attr, char *buf); ssize_t (*store)(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t count); }; extern const struct sysfs_ops kobj_sysfs_ops; struct sock; struct kset { struct list_head list; spinlock_t list_lock; struct kobject kobj; const struct kset_uevent_ops *uevent_ops; }; extern void kset_init(struct kset *kset); extern int __attribute__((warn_unused_result)) kset_register(struct kset *kset); extern void kset_unregister(struct kset *kset); extern struct kset * __attribute__((warn_unused_result)) kset_create_and_add(const char *name, const struct kset_uevent_ops *u, struct kobject *parent_kobj); static inline __attribute__((always_inline)) struct kset *to_kset(struct kobject *kobj) { return kobj ? ({ const typeof( ((struct kset *)0)->kobj ) *__mptr = (kobj); (struct kset *)( (char *)__mptr - __builtin_offsetof(struct kset,kobj) );}) : ((void *)0); } static inline __attribute__((always_inline)) struct kset *kset_get(struct kset *k) { return k ? to_kset(kobject_get(&k->kobj)) : ((void *)0); } static inline __attribute__((always_inline)) void kset_put(struct kset *k) { kobject_put(&k->kobj); } static inline __attribute__((always_inline)) struct kobj_type *get_ktype(struct kobject *kobj) { return kobj->ktype; } extern struct kobject *kset_find_obj(struct kset *, const char *); extern struct kobject *kset_find_obj_hinted(struct kset *, const char *, struct kobject *); extern struct kobject *kernel_kobj; extern struct kobject *mm_kobj; extern struct kobject *hypervisor_kobj; extern struct kobject *power_kobj; extern struct kobject *firmware_kobj; int kobject_uevent(struct kobject *kobj, enum kobject_action action); int kobject_uevent_env(struct kobject *kobj, enum kobject_action action, char *envp[]); int add_uevent_var(struct kobj_uevent_env *env, const char *format, ...) __attribute__((format (printf, 2, 3))); int kobject_action_type(const char *buf, size_t count, enum kobject_action *type); struct kernel_param; struct kernel_param_ops { int (*set)(const char *val, const struct kernel_param *kp); int (*get)(char *buffer, const struct kernel_param *kp); void (*free)(void *arg); }; struct kernel_param { const char *name; const struct kernel_param_ops *ops; u16 perm; u16 flags; union { void *arg; const struct kparam_string *str; const struct kparam_array *arr; }; }; struct kparam_string { unsigned int maxlen; char *string; }; struct kparam_array { unsigned int max; unsigned int elemsize; unsigned int *num; const struct kernel_param_ops *ops; void *elem; }; static inline __attribute__((always_inline)) int __check_old_set_param(int (*oldset)(const char *, struct kernel_param *)) { return 0; } extern void __kernel_param_lock(void); extern void __kernel_param_unlock(void); extern int parse_args(const char *name, char *args, const struct kernel_param *params, unsigned num, int (*unknown)(char *param, char *val)); extern void destroy_params(const struct kernel_param *params, unsigned num); extern struct kernel_param_ops param_ops_byte; extern int param_set_byte(const char *val, const struct kernel_param *kp); extern int param_get_byte(char *buffer, const struct kernel_param *kp); extern struct kernel_param_ops param_ops_short; extern int param_set_short(const char *val, const struct kernel_param *kp); extern int param_get_short(char *buffer, const struct kernel_param *kp); extern struct kernel_param_ops param_ops_ushort; extern int param_set_ushort(const char *val, const struct kernel_param *kp); extern int param_get_ushort(char *buffer, const struct kernel_param *kp); extern struct kernel_param_ops param_ops_int; extern int param_set_int(const char *val, const struct kernel_param *kp); extern int param_get_int(char *buffer, const struct kernel_param *kp); extern struct kernel_param_ops param_ops_uint; extern int param_set_uint(const char *val, const struct kernel_param *kp); extern int param_get_uint(char *buffer, const struct kernel_param *kp); extern struct kernel_param_ops param_ops_long; extern int param_set_long(const char *val, const struct kernel_param *kp); extern int param_get_long(char *buffer, const struct kernel_param *kp); extern struct kernel_param_ops param_ops_ulong; extern int param_set_ulong(const char *val, const struct kernel_param *kp); extern int param_get_ulong(char *buffer, const struct kernel_param *kp); extern struct kernel_param_ops param_ops_charp; extern int param_set_charp(const char *val, const struct kernel_param *kp); extern int param_get_charp(char *buffer, const struct kernel_param *kp); extern struct kernel_param_ops param_ops_bool; extern int param_set_bool(const char *val, const struct kernel_param *kp); extern int param_get_bool(char *buffer, const struct kernel_param *kp); extern struct kernel_param_ops param_ops_invbool; extern int param_set_invbool(const char *val, const struct kernel_param *kp); extern int param_get_invbool(char *buffer, const struct kernel_param *kp); extern struct kernel_param_ops param_array_ops; extern struct kernel_param_ops param_ops_string; extern int param_set_copystring(const char *val, const struct kernel_param *); extern int param_get_string(char *buffer, const struct kernel_param *kp); struct module; extern int module_param_sysfs_setup(struct module *mod, const struct kernel_param *kparam, unsigned int num_params); extern void module_param_sysfs_remove(struct module *mod); struct jump_label_key { atomic_t enabled; struct jump_entry *entries; struct jump_label_mod *next; }; static inline __attribute__((always_inline)) __attribute__((always_inline)) bool arch_static_branch(struct jump_label_key *key) { asm goto("1:" ".byte 0xe9 \n\t .long 0\n\t" ".pushsection __jump_table, \"aw\" \n\t" " " ".balign 4" " " "\n\t" " " ".long" " " "1b, %l[l_yes], %c0 \n\t" ".popsection \n\t" : : "i" (key) : : l_yes); return false; l_yes: return true; } typedef u32 jump_label_t; struct jump_entry { jump_label_t code; jump_label_t target; jump_label_t key; }; enum jump_label_type { JUMP_LABEL_DISABLE = 0, JUMP_LABEL_ENABLE, }; struct module; static inline __attribute__((always_inline)) __attribute__((always_inline)) bool static_branch(struct jump_label_key *key) { return arch_static_branch(key); } extern struct jump_entry __start___jump_table[]; extern struct jump_entry __stop___jump_table[]; extern void jump_label_lock(void); extern void jump_label_unlock(void); extern void arch_jump_label_transform(struct jump_entry *entry, enum jump_label_type type); extern void arch_jump_label_text_poke_early(jump_label_t addr); extern int jump_label_text_reserved(void *start, void *end); extern void jump_label_inc(struct jump_label_key *key); extern void jump_label_dec(struct jump_label_key *key); extern bool jump_label_enabled(struct jump_label_key *key); extern void jump_label_apply_nops(struct module *mod); struct module; struct tracepoint; struct tracepoint_func { void *func; void *data; }; struct tracepoint { const char *name; struct jump_label_key key; void (*regfunc)(void); void (*unregfunc)(void); struct tracepoint_func *funcs; }; extern int tracepoint_probe_register(const char *name, void *probe, void *data); extern int tracepoint_probe_unregister(const char *name, void *probe, void *data); extern int tracepoint_probe_register_noupdate(const char *name, void *probe, void *data); extern int tracepoint_probe_unregister_noupdate(const char *name, void *probe, void *data); extern void tracepoint_probe_update_all(void); struct tracepoint_iter { struct module *module; struct tracepoint * const *tracepoint; }; extern void tracepoint_iter_start(struct tracepoint_iter *iter); extern void tracepoint_iter_next(struct tracepoint_iter *iter); extern void tracepoint_iter_stop(struct tracepoint_iter *iter); extern void tracepoint_iter_reset(struct tracepoint_iter *iter); extern int tracepoint_get_iter_range(struct tracepoint * const **tracepoint, struct tracepoint * const *begin, struct tracepoint * const *end); static inline __attribute__((always_inline)) void tracepoint_synchronize_unregister(void) { synchronize_sched(); } extern void tracepoint_update_probe_range(struct tracepoint * const *begin, struct tracepoint * const *end); struct mod_arch_specific { }; struct module; extern struct tracepoint __tracepoint_module_load ; static inline __attribute__((always_inline)) void trace_module_load (struct module *mod) { if (__builtin_constant_p(((static_branch(&__tracepoint_module_load.key)))) ? !!((static_branch(&__tracepoint_module_load.key))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/trace/events/module.h" , .line = 45 , }; ______r = !!((static_branch(&__tracepoint_module_load.key))); ______f.miss_hit[______r]++; ______r; })) do { struct tracepoint_func *it_func_ptr; void *it_func; void *__data; if (__builtin_constant_p(((!(1)))) ? !!((!(1))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/trace/events/module.h" , .line = 45 , }; ______r = !!((!(1))); ______f.miss_hit[______r]++; ______r; })) return; rcu_read_lock_sched_notrace(); it_func_ptr = ({ typeof(*((&__tracepoint_module_load)->funcs)) *_________p1 = (typeof(*((&__tracepoint_module_load)->funcs))* )(*(volatile typeof(((&__tracepoint_module_load)->funcs)) *)&(((&__tracepoint_module_load)->funcs))); do { } while (0); ; do { } while (0); ((typeof(*((&__tracepoint_module_load)->funcs)) *)(_________p1)); }); if (__builtin_constant_p(((it_func_ptr))) ? !!((it_func_ptr)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/trace/events/module.h" , .line = 45 , }; ______r = !!((it_func_ptr)); ______f.miss_hit[______r]++; ______r; })) { do { it_func = (it_func_ptr)->func; __data = (it_func_ptr)->data; ((void(*)(void *__data, struct module *mod))(it_func))(__data, mod); } while ((++it_func_ptr)->func); } rcu_read_unlock_sched_notrace(); } while (0); } static inline __attribute__((always_inline)) int register_trace_module_load (void (*probe)(void *__data, struct module *mod), void *data) { return tracepoint_probe_register("module_load", (void *)probe, data); } static inline __attribute__((always_inline)) int unregister_trace_module_load (void (*probe)(void *__data, struct module *mod), void *data) { return tracepoint_probe_unregister("module_load", (void *)probe, data); } static inline __attribute__((always_inline)) void check_trace_callback_type_module_load (void (*cb)(void *__data, struct module *mod)) { } ; extern struct tracepoint __tracepoint_module_free ; static inline __attribute__((always_inline)) void trace_module_free (struct module *mod) { if (__builtin_constant_p(((static_branch(&__tracepoint_module_free.key)))) ? !!((static_branch(&__tracepoint_module_free.key))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/trace/events/module.h" , .line = 62 , }; ______r = !!((static_branch(&__tracepoint_module_free.key))); ______f.miss_hit[______r]++; ______r; })) do { struct tracepoint_func *it_func_ptr; void *it_func; void *__data; if (__builtin_constant_p(((!(1)))) ? !!((!(1))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/trace/events/module.h" , .line = 62 , }; ______r = !!((!(1))); ______f.miss_hit[______r]++; ______r; })) return; rcu_read_lock_sched_notrace(); it_func_ptr = ({ typeof(*((&__tracepoint_module_free)->funcs)) *_________p1 = (typeof(*((&__tracepoint_module_free)->funcs))* )(*(volatile typeof(((&__tracepoint_module_free)->funcs)) *)&(((&__tracepoint_module_free)->funcs))); do { } while (0); ; do { } while (0); ((typeof(*((&__tracepoint_module_free)->funcs)) *)(_________p1)); }); if (__builtin_constant_p(((it_func_ptr))) ? !!((it_func_ptr)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/trace/events/module.h" , .line = 62 , }; ______r = !!((it_func_ptr)); ______f.miss_hit[______r]++; ______r; })) { do { it_func = (it_func_ptr)->func; __data = (it_func_ptr)->data; ((void(*)(void *__data, struct module *mod))(it_func))(__data, mod); } while ((++it_func_ptr)->func); } rcu_read_unlock_sched_notrace(); } while (0); } static inline __attribute__((always_inline)) int register_trace_module_free (void (*probe)(void *__data, struct module *mod), void *data) { return tracepoint_probe_register("module_free", (void *)probe, data); } static inline __attribute__((always_inline)) int unregister_trace_module_free (void (*probe)(void *__data, struct module *mod), void *data) { return tracepoint_probe_unregister("module_free", (void *)probe, data); } static inline __attribute__((always_inline)) void check_trace_callback_type_module_free (void (*cb)(void *__data, struct module *mod)) { } ; ; extern struct tracepoint __tracepoint_module_get ; static inline __attribute__((always_inline)) void trace_module_get (struct module *mod, unsigned long ip) { if (__builtin_constant_p(((static_branch(&__tracepoint_module_get.key)))) ? !!((static_branch(&__tracepoint_module_get.key))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/trace/events/module.h" , .line = 94 , }; ______r = !!((static_branch(&__tracepoint_module_get.key))); ______f.miss_hit[______r]++; ______r; })) do { struct tracepoint_func *it_func_ptr; void *it_func; void *__data; if (__builtin_constant_p(((!(1)))) ? !!((!(1))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/trace/events/module.h" , .line = 94 , }; ______r = !!((!(1))); ______f.miss_hit[______r]++; ______r; })) return; rcu_read_lock_sched_notrace(); it_func_ptr = ({ typeof(*((&__tracepoint_module_get)->funcs)) *_________p1 = (typeof(*((&__tracepoint_module_get)->funcs))* )(*(volatile typeof(((&__tracepoint_module_get)->funcs)) *)&(((&__tracepoint_module_get)->funcs))); do { } while (0); ; do { } while (0); ((typeof(*((&__tracepoint_module_get)->funcs)) *)(_________p1)); }); if (__builtin_constant_p(((it_func_ptr))) ? !!((it_func_ptr)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/trace/events/module.h" , .line = 94 , }; ______r = !!((it_func_ptr)); ______f.miss_hit[______r]++; ______r; })) { do { it_func = (it_func_ptr)->func; __data = (it_func_ptr)->data; ((void(*)(void *__data, struct module *mod, unsigned long ip))(it_func))(__data, mod, ip); } while ((++it_func_ptr)->func); } rcu_read_unlock_sched_notrace(); } while (0); } static inline __attribute__((always_inline)) int register_trace_module_get (void (*probe)(void *__data, struct module *mod, unsigned long ip), void *data) { return tracepoint_probe_register("module_get", (void *)probe, data); } static inline __attribute__((always_inline)) int unregister_trace_module_get (void (*probe)(void *__data, struct module *mod, unsigned long ip), void *data) { return tracepoint_probe_unregister("module_get", (void *)probe, data); } static inline __attribute__((always_inline)) void check_trace_callback_type_module_get (void (*cb)(void *__data, struct module *mod, unsigned long ip)) { } ; extern struct tracepoint __tracepoint_module_put ; static inline __attribute__((always_inline)) void trace_module_put (struct module *mod, unsigned long ip) { if (__builtin_constant_p(((static_branch(&__tracepoint_module_put.key)))) ? !!((static_branch(&__tracepoint_module_put.key))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/trace/events/module.h" , .line = 101 , }; ______r = !!((static_branch(&__tracepoint_module_put.key))); ______f.miss_hit[______r]++; ______r; })) do { struct tracepoint_func *it_func_ptr; void *it_func; void *__data; if (__builtin_constant_p(((!(1)))) ? !!((!(1))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/trace/events/module.h" , .line = 101 , }; ______r = !!((!(1))); ______f.miss_hit[______r]++; ______r; })) return; rcu_read_lock_sched_notrace(); it_func_ptr = ({ typeof(*((&__tracepoint_module_put)->funcs)) *_________p1 = (typeof(*((&__tracepoint_module_put)->funcs))* )(*(volatile typeof(((&__tracepoint_module_put)->funcs)) *)&(((&__tracepoint_module_put)->funcs))); do { } while (0); ; do { } while (0); ((typeof(*((&__tracepoint_module_put)->funcs)) *)(_________p1)); }); if (__builtin_constant_p(((it_func_ptr))) ? !!((it_func_ptr)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/trace/events/module.h" , .line = 101 , }; ______r = !!((it_func_ptr)); ______f.miss_hit[______r]++; ______r; })) { do { it_func = (it_func_ptr)->func; __data = (it_func_ptr)->data; ((void(*)(void *__data, struct module *mod, unsigned long ip))(it_func))(__data, mod, ip); } while ((++it_func_ptr)->func); } rcu_read_unlock_sched_notrace(); } while (0); } static inline __attribute__((always_inline)) int register_trace_module_put (void (*probe)(void *__data, struct module *mod, unsigned long ip), void *data) { return tracepoint_probe_register("module_put", (void *)probe, data); } static inline __attribute__((always_inline)) int unregister_trace_module_put (void (*probe)(void *__data, struct module *mod, unsigned long ip), void *data) { return tracepoint_probe_unregister("module_put", (void *)probe, data); } static inline __attribute__((always_inline)) void check_trace_callback_type_module_put (void (*cb)(void *__data, struct module *mod, unsigned long ip)) { } ; extern struct tracepoint __tracepoint_module_request ; static inline __attribute__((always_inline)) void trace_module_request (char *name, bool wait, unsigned long ip) { if (__builtin_constant_p(((static_branch(&__tracepoint_module_request.key)))) ? !!((static_branch(&__tracepoint_module_request.key))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/trace/events/module.h" , .line = 124 , }; ______r = !!((static_branch(&__tracepoint_module_request.key))); ______f.miss_hit[______r]++; ______r; })) do { struct tracepoint_func *it_func_ptr; void *it_func; void *__data; if (__builtin_constant_p(((!(1)))) ? !!((!(1))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/trace/events/module.h" , .line = 124 , }; ______r = !!((!(1))); ______f.miss_hit[______r]++; ______r; })) return; rcu_read_lock_sched_notrace(); it_func_ptr = ({ typeof(*((&__tracepoint_module_request)->funcs)) *_________p1 = (typeof(*((&__tracepoint_module_request)->funcs))* )(*(volatile typeof(((&__tracepoint_module_request)->funcs)) *)&(((&__tracepoint_module_request)->funcs))); do { } while (0); ; do { } while (0); ((typeof(*((&__tracepoint_module_request)->funcs)) *)(_________p1)); }); if (__builtin_constant_p(((it_func_ptr))) ? !!((it_func_ptr)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/trace/events/module.h" , .line = 124 , }; ______r = !!((it_func_ptr)); ______f.miss_hit[______r]++; ______r; })) { do { it_func = (it_func_ptr)->func; __data = (it_func_ptr)->data; ((void(*)(void *__data, char *name, bool wait, unsigned long ip))(it_func))(__data, name, wait, ip); } while ((++it_func_ptr)->func); } rcu_read_unlock_sched_notrace(); } while (0); } static inline __attribute__((always_inline)) int register_trace_module_request (void (*probe)(void *__data, char *name, bool wait, unsigned long ip), void *data) { return tracepoint_probe_register("module_request", (void *)probe, data); } static inline __attribute__((always_inline)) int unregister_trace_module_request (void (*probe)(void *__data, char *name, bool wait, unsigned long ip), void *data) { return tracepoint_probe_unregister("module_request", (void *)probe, data); } static inline __attribute__((always_inline)) void check_trace_callback_type_module_request (void (*cb)(void *__data, char *name, bool wait, unsigned long ip)) { } ; struct kernel_symbol { unsigned long value; const char *name; }; struct modversion_info { unsigned long crc; char name[(64 - sizeof(unsigned long))]; }; struct module; struct module_attribute { struct attribute attr; ssize_t (*show)(struct module_attribute *, struct module *, char *); ssize_t (*store)(struct module_attribute *, struct module *, const char *, size_t count); void (*setup)(struct module *, const char *); int (*test)(struct module *); void (*free)(struct module *); }; struct module_version_attribute { struct module_attribute mattr; const char *module_name; const char *version; } __attribute__ ((__aligned__(sizeof(void *)))); extern ssize_t __modver_version_show(struct module_attribute *, struct module *, char *); struct module_kobject { struct kobject kobj; struct module *mod; struct kobject *drivers_dir; struct module_param_attrs *mp; }; extern int init_module(void); extern void cleanup_module(void); struct exception_table_entry; const struct exception_table_entry * search_extable(const struct exception_table_entry *first, const struct exception_table_entry *last, unsigned long value); void sort_extable(struct exception_table_entry *start, struct exception_table_entry *finish); void sort_main_extable(void); void trim_init_extable(struct module *m); extern struct module __this_module; const struct exception_table_entry *search_exception_tables(unsigned long add); struct notifier_block; extern int modules_disabled; void *__symbol_get(const char *symbol); void *__symbol_get_gpl(const char *symbol); struct module_use { struct list_head source_list; struct list_head target_list; struct module *source, *target; }; enum module_state { MODULE_STATE_LIVE, MODULE_STATE_COMING, MODULE_STATE_GOING, }; struct module { enum module_state state; struct list_head list; char name[(64 - sizeof(unsigned long))]; struct module_kobject mkobj; struct module_attribute *modinfo_attrs; const char *version; const char *srcversion; struct kobject *holders_dir; const struct kernel_symbol *syms; const unsigned long *crcs; unsigned int num_syms; struct kernel_param *kp; unsigned int num_kp; unsigned int num_gpl_syms; const struct kernel_symbol *gpl_syms; const unsigned long *gpl_crcs; const struct kernel_symbol *unused_syms; const unsigned long *unused_crcs; unsigned int num_unused_syms; unsigned int num_unused_gpl_syms; const struct kernel_symbol *unused_gpl_syms; const unsigned long *unused_gpl_crcs; const struct kernel_symbol *gpl_future_syms; const unsigned long *gpl_future_crcs; unsigned int num_gpl_future_syms; unsigned int num_exentries; struct exception_table_entry *extable; int (*init)(void); void *module_init; void *module_core; unsigned int init_size, core_size; unsigned int init_text_size, core_text_size; unsigned int init_ro_size, core_ro_size; struct mod_arch_specific arch; unsigned int taints; unsigned num_bugs; struct list_head bug_list; struct bug_entry *bug_table; Elf32_Sym *symtab, *core_symtab; unsigned int num_symtab, core_num_syms; char *strtab, *core_strtab; struct module_sect_attrs *sect_attrs; struct module_notes_attrs *notes_attrs; char *args; void *percpu; unsigned int percpu_size; unsigned int num_tracepoints; struct tracepoint * const *tracepoints_ptrs; struct jump_entry *jump_entries; unsigned int num_jump_entries; unsigned int num_trace_bprintk_fmt; const char **trace_bprintk_fmt_start; struct ftrace_event_call **trace_events; unsigned int num_trace_events; unsigned int num_ftrace_callsites; unsigned long *ftrace_callsites; struct list_head source_list; struct list_head target_list; struct task_struct *waiter; void (*exit)(void); struct module_ref { unsigned int incs; unsigned int decs; } *refptr; }; extern struct mutex module_mutex; static inline __attribute__((always_inline)) int module_is_live(struct module *mod) { return mod->state != MODULE_STATE_GOING; } struct module *__module_text_address(unsigned long addr); struct module *__module_address(unsigned long addr); bool is_module_address(unsigned long addr); bool is_module_percpu_address(unsigned long addr); bool is_module_text_address(unsigned long addr); static inline __attribute__((always_inline)) int within_module_core(unsigned long addr, struct module *mod) { return (unsigned long)mod->module_core <= addr && addr < (unsigned long)mod->module_core + mod->core_size; } static inline __attribute__((always_inline)) int within_module_init(unsigned long addr, struct module *mod) { return (unsigned long)mod->module_init <= addr && addr < (unsigned long)mod->module_init + mod->init_size; } struct module *find_module(const char *name); struct symsearch { const struct kernel_symbol *start, *stop; const unsigned long *crcs; enum { NOT_GPL_ONLY, GPL_ONLY, WILL_BE_GPL_ONLY, } licence; bool unused; }; const struct kernel_symbol *find_symbol(const char *name, struct module **owner, const unsigned long **crc, bool gplok, bool warn); bool each_symbol_section(bool (*fn)(const struct symsearch *arr, struct module *owner, void *data), void *data); int module_get_kallsym(unsigned int symnum, unsigned long *value, char *type, char *name, char *module_name, int *exported); unsigned long module_kallsyms_lookup_name(const char *name); int module_kallsyms_on_each_symbol(int (*fn)(void *, const char *, struct module *, unsigned long), void *data); extern void __module_put_and_exit(struct module *mod, long code) __attribute__((noreturn)); unsigned int module_refcount(struct module *mod); void __symbol_put(const char *symbol); void symbol_put_addr(void *addr); static inline __attribute__((always_inline)) void __module_get(struct module *module) { if (__builtin_constant_p(((module))) ? !!((module)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 510, }; ______r = !!((module)); ______f.miss_hit[______r]++; ______r; })) { do { add_preempt_count(1); __asm__ __volatile__("": : :"memory"); } while (0); do { do { const void *__vpp_verify = (typeof(&(((module->refptr->incs)))))((void *)0); (void)__vpp_verify; } while (0); switch(sizeof(((module->refptr->incs)))) { case 1: do { typedef typeof((((module->refptr->incs)))) pao_T__; const int pao_ID__ = (__builtin_constant_p((1)) && (((1)) == 1 || ((1)) == -1)) ? ((1)) : 0; if (__builtin_constant_p(((0))) ? !!((0)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 512, }; ______r = !!((0)); ______f.miss_hit[______r]++; ______r; })) { pao_T__ pao_tmp__; pao_tmp__ = ((1)); (void)pao_tmp__; } switch (sizeof((((module->refptr->incs))))) { case 1: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 512, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incb ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 512, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decb ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else asm("addb %1, ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs)))) : "qi" ((pao_T__)((1)))); break; case 2: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 512, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incw ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 512, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decw ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else asm("addw %1, ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs)))) : "ri" ((pao_T__)((1)))); break; case 4: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 512, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incl ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 512, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decl ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else asm("addl %1, ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs)))) : "ri" ((pao_T__)((1)))); break; case 8: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 512, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incq ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 512, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decq ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else asm("addq %1, ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs)))) : "re" ((pao_T__)((1)))); break; default: __bad_percpu_size(); } } while (0);break; case 2: do { typedef typeof((((module->refptr->incs)))) pao_T__; const int pao_ID__ = (__builtin_constant_p((1)) && (((1)) == 1 || ((1)) == -1)) ? ((1)) : 0; if (__builtin_constant_p(((0))) ? !!((0)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 512, }; ______r = !!((0)); ______f.miss_hit[______r]++; ______r; })) { pao_T__ pao_tmp__; pao_tmp__ = ((1)); (void)pao_tmp__; } switch (sizeof((((module->refptr->incs))))) { case 1: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 512, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incb ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 512, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decb ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else asm("addb %1, ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs)))) : "qi" ((pao_T__)((1)))); break; case 2: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 512, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incw ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 512, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decw ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else asm("addw %1, ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs)))) : "ri" ((pao_T__)((1)))); break; case 4: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 512, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incl ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 512, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decl ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else asm("addl %1, ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs)))) : "ri" ((pao_T__)((1)))); break; case 8: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 512, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incq ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 512, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decq ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else asm("addq %1, ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs)))) : "re" ((pao_T__)((1)))); break; default: __bad_percpu_size(); } } while (0);break; case 4: do { typedef typeof((((module->refptr->incs)))) pao_T__; const int pao_ID__ = (__builtin_constant_p((1)) && (((1)) == 1 || ((1)) == -1)) ? ((1)) : 0; if (__builtin_constant_p(((0))) ? !!((0)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 512, }; ______r = !!((0)); ______f.miss_hit[______r]++; ______r; })) { pao_T__ pao_tmp__; pao_tmp__ = ((1)); (void)pao_tmp__; } switch (sizeof((((module->refptr->incs))))) { case 1: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 512, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incb ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 512, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decb ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else asm("addb %1, ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs)))) : "qi" ((pao_T__)((1)))); break; case 2: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 512, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incw ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 512, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decw ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else asm("addw %1, ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs)))) : "ri" ((pao_T__)((1)))); break; case 4: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 512, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incl ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 512, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decl ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else asm("addl %1, ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs)))) : "ri" ((pao_T__)((1)))); break; case 8: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 512, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incq ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 512, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decq ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else asm("addq %1, ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs)))) : "re" ((pao_T__)((1)))); break; default: __bad_percpu_size(); } } while (0);break; case 8: do { *({ unsigned long tcp_ptr__; do { const void *__vpp_verify = (typeof(&((((module->refptr->incs))))))((void *)0); (void)__vpp_verify; } while (0); asm volatile("add " "%%""fs"":" "%P" "1" ", %0" : "=r" (tcp_ptr__) : "m" (this_cpu_off), "0" (&((((module->refptr->incs)))))); (typeof(*(&((((module->refptr->incs)))))) *)tcp_ptr__; }) += ((1)); } while (0);break; default: __bad_size_call_parameter();break; } } while (0); trace_module_get(module, ({ __label__ __here; __here: (unsigned long)&&__here; })); do { do { __asm__ __volatile__("": : :"memory"); sub_preempt_count(1); } while (0); __asm__ __volatile__("": : :"memory"); do { if (__builtin_constant_p((((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 514, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 514, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 514, }; ______r = !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 514, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) preempt_schedule(); } while (0); } while (0); } } static inline __attribute__((always_inline)) int try_module_get(struct module *module) { int ret = 1; if (__builtin_constant_p(((module))) ? !!((module)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 522, }; ______r = !!((module)); ______f.miss_hit[______r]++; ______r; })) { do { add_preempt_count(1); __asm__ __volatile__("": : :"memory"); } while (0); if (__builtin_constant_p((((__builtin_constant_p(module_is_live(module)) ? !!(module_is_live(module)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 525, }; ______r = __builtin_expect(!!(module_is_live(module)), 1); ftrace_likely_update(&______f, ______r, 1); ______r; }))))) ? !!(((__builtin_constant_p(module_is_live(module)) ? !!(module_is_live(module)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 525, }; ______r = __builtin_expect(!!(module_is_live(module)), 1); ftrace_likely_update(&______f, ______r, 1); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 525, }; ______r = !!(((__builtin_constant_p(module_is_live(module)) ? !!(module_is_live(module)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 525, }; ______r = __builtin_expect(!!(module_is_live(module)), 1); ftrace_likely_update(&______f, ______r, 1); ______r; })))); ______f.miss_hit[______r]++; ______r; })) { do { do { const void *__vpp_verify = (typeof(&(((module->refptr->incs)))))((void *)0); (void)__vpp_verify; } while (0); switch(sizeof(((module->refptr->incs)))) { case 1: do { typedef typeof((((module->refptr->incs)))) pao_T__; const int pao_ID__ = (__builtin_constant_p((1)) && (((1)) == 1 || ((1)) == -1)) ? ((1)) : 0; if (__builtin_constant_p(((0))) ? !!((0)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 526, }; ______r = !!((0)); ______f.miss_hit[______r]++; ______r; })) { pao_T__ pao_tmp__; pao_tmp__ = ((1)); (void)pao_tmp__; } switch (sizeof((((module->refptr->incs))))) { case 1: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 526, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incb ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 526, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decb ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else asm("addb %1, ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs)))) : "qi" ((pao_T__)((1)))); break; case 2: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 526, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incw ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 526, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decw ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else asm("addw %1, ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs)))) : "ri" ((pao_T__)((1)))); break; case 4: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 526, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incl ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 526, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decl ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else asm("addl %1, ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs)))) : "ri" ((pao_T__)((1)))); break; case 8: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 526, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incq ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 526, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decq ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else asm("addq %1, ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs)))) : "re" ((pao_T__)((1)))); break; default: __bad_percpu_size(); } } while (0);break; case 2: do { typedef typeof((((module->refptr->incs)))) pao_T__; const int pao_ID__ = (__builtin_constant_p((1)) && (((1)) == 1 || ((1)) == -1)) ? ((1)) : 0; if (__builtin_constant_p(((0))) ? !!((0)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 526, }; ______r = !!((0)); ______f.miss_hit[______r]++; ______r; })) { pao_T__ pao_tmp__; pao_tmp__ = ((1)); (void)pao_tmp__; } switch (sizeof((((module->refptr->incs))))) { case 1: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 526, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incb ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 526, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decb ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else asm("addb %1, ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs)))) : "qi" ((pao_T__)((1)))); break; case 2: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 526, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incw ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 526, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decw ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else asm("addw %1, ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs)))) : "ri" ((pao_T__)((1)))); break; case 4: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 526, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incl ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 526, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decl ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else asm("addl %1, ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs)))) : "ri" ((pao_T__)((1)))); break; case 8: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 526, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incq ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 526, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decq ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else asm("addq %1, ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs)))) : "re" ((pao_T__)((1)))); break; default: __bad_percpu_size(); } } while (0);break; case 4: do { typedef typeof((((module->refptr->incs)))) pao_T__; const int pao_ID__ = (__builtin_constant_p((1)) && (((1)) == 1 || ((1)) == -1)) ? ((1)) : 0; if (__builtin_constant_p(((0))) ? !!((0)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 526, }; ______r = !!((0)); ______f.miss_hit[______r]++; ______r; })) { pao_T__ pao_tmp__; pao_tmp__ = ((1)); (void)pao_tmp__; } switch (sizeof((((module->refptr->incs))))) { case 1: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 526, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incb ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 526, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decb ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else asm("addb %1, ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs)))) : "qi" ((pao_T__)((1)))); break; case 2: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 526, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incw ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 526, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decw ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else asm("addw %1, ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs)))) : "ri" ((pao_T__)((1)))); break; case 4: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 526, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incl ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 526, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decl ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else asm("addl %1, ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs)))) : "ri" ((pao_T__)((1)))); break; case 8: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 526, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incq ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 526, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decq ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs))))); else asm("addq %1, ""%%""fs"":" "%P" "0" : "+m" ((((module->refptr->incs)))) : "re" ((pao_T__)((1)))); break; default: __bad_percpu_size(); } } while (0);break; case 8: do { *({ unsigned long tcp_ptr__; do { const void *__vpp_verify = (typeof(&((((module->refptr->incs))))))((void *)0); (void)__vpp_verify; } while (0); asm volatile("add " "%%""fs"":" "%P" "1" ", %0" : "=r" (tcp_ptr__) : "m" (this_cpu_off), "0" (&((((module->refptr->incs)))))); (typeof(*(&((((module->refptr->incs)))))) *)tcp_ptr__; }) += ((1)); } while (0);break; default: __bad_size_call_parameter();break; } } while (0); trace_module_get(module, ({ __label__ __here; __here: (unsigned long)&&__here; })); } else ret = 0; do { do { __asm__ __volatile__("": : :"memory"); sub_preempt_count(1); } while (0); __asm__ __volatile__("": : :"memory"); do { if (__builtin_constant_p((((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 531, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 531, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 531, }; ______r = !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/module.h", .line = 531, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) preempt_schedule(); } while (0); } while (0); } return ret; } extern void module_put(struct module *module); int ref_module(struct module *a, struct module *b); const char *module_address_lookup(unsigned long addr, unsigned long *symbolsize, unsigned long *offset, char **modname, char *namebuf); int lookup_module_symbol_name(unsigned long addr, char *symname); int lookup_module_symbol_attrs(unsigned long addr, unsigned long *size, unsigned long *offset, char *modname, char *name); const struct exception_table_entry *search_module_extables(unsigned long addr); int register_module_notifier(struct notifier_block * nb); int unregister_module_notifier(struct notifier_block * nb); extern void print_modules(void); extern void module_update_tracepoints(void); extern int module_get_iter_tracepoints(struct tracepoint_iter *iter); extern struct kset *module_kset; extern struct kobj_type module_ktype; extern int module_sysfs_initialized; static inline __attribute__((always_inline)) void set_all_modules_text_rw(void) { } static inline __attribute__((always_inline)) void set_all_modules_text_ro(void) { } void module_bug_finalize(const Elf32_Ehdr *, const Elf32_Shdr *, struct module *); void module_bug_cleanup(struct module *); void __attribute__ ((__section__(".init.text"))) __attribute__((__cold__)) __attribute__((no_instrument_function)) kmem_cache_init(void); int slab_is_available(void); struct kmem_cache *kmem_cache_create(const char *, size_t, size_t, unsigned long, void (*)(void *)); void kmem_cache_destroy(struct kmem_cache *); int kmem_cache_shrink(struct kmem_cache *); void kmem_cache_free(struct kmem_cache *, void *); unsigned int kmem_cache_size(struct kmem_cache *); void * __attribute__((warn_unused_result)) __krealloc(const void *, size_t, gfp_t); void * __attribute__((warn_unused_result)) krealloc(const void *, size_t, gfp_t); void kfree(const void *); void kzfree(const void *); size_t ksize(const void *); static inline __attribute__((always_inline)) void kmemleak_init(void) { } static inline __attribute__((always_inline)) void kmemleak_alloc(const void *ptr, size_t size, int min_count, gfp_t gfp) { } static inline __attribute__((always_inline)) void kmemleak_alloc_recursive(const void *ptr, size_t size, int min_count, unsigned long flags, gfp_t gfp) { } static inline __attribute__((always_inline)) void kmemleak_free(const void *ptr) { } static inline __attribute__((always_inline)) void kmemleak_free_part(const void *ptr, size_t size) { } static inline __attribute__((always_inline)) void kmemleak_free_recursive(const void *ptr, unsigned long flags) { } static inline __attribute__((always_inline)) void kmemleak_not_leak(const void *ptr) { } static inline __attribute__((always_inline)) void kmemleak_ignore(const void *ptr) { } static inline __attribute__((always_inline)) void kmemleak_scan_area(const void *ptr, size_t size, gfp_t gfp) { } static inline __attribute__((always_inline)) void kmemleak_erase(void **ptr) { } static inline __attribute__((always_inline)) void kmemleak_no_scan(const void *ptr) { } enum stat_item { ALLOC_FASTPATH, ALLOC_SLOWPATH, FREE_FASTPATH, FREE_SLOWPATH, FREE_FROZEN, FREE_ADD_PARTIAL, FREE_REMOVE_PARTIAL, ALLOC_FROM_PARTIAL, ALLOC_SLAB, ALLOC_REFILL, FREE_SLAB, CPUSLAB_FLUSH, DEACTIVATE_FULL, DEACTIVATE_EMPTY, DEACTIVATE_TO_HEAD, DEACTIVATE_TO_TAIL, DEACTIVATE_REMOTE_FREES, ORDER_FALLBACK, CMPXCHG_DOUBLE_CPU_FAIL, NR_SLUB_STAT_ITEMS }; struct kmem_cache_cpu { void **freelist; unsigned long tid; struct page *page; int node; }; struct kmem_cache_node { spinlock_t list_lock; unsigned long nr_partial; struct list_head partial; atomic_long_t nr_slabs; atomic_long_t total_objects; struct list_head full; }; struct kmem_cache_order_objects { unsigned long x; }; struct kmem_cache { struct kmem_cache_cpu *cpu_slab; unsigned long flags; unsigned long min_partial; int size; int objsize; int offset; struct kmem_cache_order_objects oo; struct kmem_cache_order_objects max; struct kmem_cache_order_objects min; gfp_t allocflags; int refcount; void (*ctor)(void *); int inuse; int align; int reserved; const char *name; struct list_head list; struct kobject kobj; struct kmem_cache_node *node[(1 << 0)]; }; extern struct kmem_cache *kmalloc_caches[(12 + 2)]; static inline __attribute__((always_inline)) __attribute__((always_inline)) int kmalloc_index(size_t size) { if (__builtin_constant_p(((!size))) ? !!((!size)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/slub_def.h", .line = 158, }; ______r = !!((!size)); ______f.miss_hit[______r]++; ______r; })) return 0; if (__builtin_constant_p(((size <= 8))) ? !!((size <= 8)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/slub_def.h", .line = 161, }; ______r = !!((size <= 8)); ______f.miss_hit[______r]++; ______r; })) return ( __builtin_constant_p(8) ? ( (8) < 1 ? ____ilog2_NaN() : (8) & (1ULL << 63) ? 63 : (8) & (1ULL << 62) ? 62 : (8) & (1ULL << 61) ? 61 : (8) & (1ULL << 60) ? 60 : (8) & (1ULL << 59) ? 59 : (8) & (1ULL << 58) ? 58 : (8) & (1ULL << 57) ? 57 : (8) & (1ULL << 56) ? 56 : (8) & (1ULL << 55) ? 55 : (8) & (1ULL << 54) ? 54 : (8) & (1ULL << 53) ? 53 : (8) & (1ULL << 52) ? 52 : (8) & (1ULL << 51) ? 51 : (8) & (1ULL << 50) ? 50 : (8) & (1ULL << 49) ? 49 : (8) & (1ULL << 48) ? 48 : (8) & (1ULL << 47) ? 47 : (8) & (1ULL << 46) ? 46 : (8) & (1ULL << 45) ? 45 : (8) & (1ULL << 44) ? 44 : (8) & (1ULL << 43) ? 43 : (8) & (1ULL << 42) ? 42 : (8) & (1ULL << 41) ? 41 : (8) & (1ULL << 40) ? 40 : (8) & (1ULL << 39) ? 39 : (8) & (1ULL << 38) ? 38 : (8) & (1ULL << 37) ? 37 : (8) & (1ULL << 36) ? 36 : (8) & (1ULL << 35) ? 35 : (8) & (1ULL << 34) ? 34 : (8) & (1ULL << 33) ? 33 : (8) & (1ULL << 32) ? 32 : (8) & (1ULL << 31) ? 31 : (8) & (1ULL << 30) ? 30 : (8) & (1ULL << 29) ? 29 : (8) & (1ULL << 28) ? 28 : (8) & (1ULL << 27) ? 27 : (8) & (1ULL << 26) ? 26 : (8) & (1ULL << 25) ? 25 : (8) & (1ULL << 24) ? 24 : (8) & (1ULL << 23) ? 23 : (8) & (1ULL << 22) ? 22 : (8) & (1ULL << 21) ? 21 : (8) & (1ULL << 20) ? 20 : (8) & (1ULL << 19) ? 19 : (8) & (1ULL << 18) ? 18 : (8) & (1ULL << 17) ? 17 : (8) & (1ULL << 16) ? 16 : (8) & (1ULL << 15) ? 15 : (8) & (1ULL << 14) ? 14 : (8) & (1ULL << 13) ? 13 : (8) & (1ULL << 12) ? 12 : (8) & (1ULL << 11) ? 11 : (8) & (1ULL << 10) ? 10 : (8) & (1ULL << 9) ? 9 : (8) & (1ULL << 8) ? 8 : (8) & (1ULL << 7) ? 7 : (8) & (1ULL << 6) ? 6 : (8) & (1ULL << 5) ? 5 : (8) & (1ULL << 4) ? 4 : (8) & (1ULL << 3) ? 3 : (8) & (1ULL << 2) ? 2 : (8) & (1ULL << 1) ? 1 : (8) & (1ULL << 0) ? 0 : ____ilog2_NaN() ) : (sizeof(8) <= 4) ? __ilog2_u32(8) : __ilog2_u64(8) ); if (__builtin_constant_p(((8 <= 32 && size > 64 && size <= 96))) ? !!((8 <= 32 && size > 64 && size <= 96)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/slub_def.h", .line = 164, }; ______r = !!((8 <= 32 && size > 64 && size <= 96)); ______f.miss_hit[______r]++; ______r; })) return 1; if (__builtin_constant_p(((8 <= 64 && size > 128 && size <= 192))) ? !!((8 <= 64 && size > 128 && size <= 192)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/slub_def.h", .line = 166, }; ______r = !!((8 <= 64 && size > 128 && size <= 192)); ______f.miss_hit[______r]++; ______r; })) return 2; if (__builtin_constant_p(((size <= 8))) ? !!((size <= 8)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/slub_def.h", .line = 168, }; ______r = !!((size <= 8)); ______f.miss_hit[______r]++; ______r; })) return 3; if (__builtin_constant_p(((size <= 16))) ? !!((size <= 16)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/slub_def.h", .line = 169, }; ______r = !!((size <= 16)); ______f.miss_hit[______r]++; ______r; })) return 4; if (__builtin_constant_p(((size <= 32))) ? !!((size <= 32)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/slub_def.h", .line = 170, }; ______r = !!((size <= 32)); ______f.miss_hit[______r]++; ______r; })) return 5; if (__builtin_constant_p(((size <= 64))) ? !!((size <= 64)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/slub_def.h", .line = 171, }; ______r = !!((size <= 64)); ______f.miss_hit[______r]++; ______r; })) return 6; if (__builtin_constant_p(((size <= 128))) ? !!((size <= 128)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/slub_def.h", .line = 172, }; ______r = !!((size <= 128)); ______f.miss_hit[______r]++; ______r; })) return 7; if (__builtin_constant_p(((size <= 256))) ? !!((size <= 256)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/slub_def.h", .line = 173, }; ______r = !!((size <= 256)); ______f.miss_hit[______r]++; ______r; })) return 8; if (__builtin_constant_p(((size <= 512))) ? !!((size <= 512)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/slub_def.h", .line = 174, }; ______r = !!((size <= 512)); ______f.miss_hit[______r]++; ______r; })) return 9; if (__builtin_constant_p(((size <= 1024))) ? !!((size <= 1024)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/slub_def.h", .line = 175, }; ______r = !!((size <= 1024)); ______f.miss_hit[______r]++; ______r; })) return 10; if (__builtin_constant_p(((size <= 2 * 1024))) ? !!((size <= 2 * 1024)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/slub_def.h", .line = 176, }; ______r = !!((size <= 2 * 1024)); ______f.miss_hit[______r]++; ______r; })) return 11; if (__builtin_constant_p(((size <= 4 * 1024))) ? !!((size <= 4 * 1024)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/slub_def.h", .line = 177, }; ______r = !!((size <= 4 * 1024)); ______f.miss_hit[______r]++; ______r; })) return 12; if (__builtin_constant_p(((size <= 8 * 1024))) ? !!((size <= 8 * 1024)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/slub_def.h", .line = 183, }; ______r = !!((size <= 8 * 1024)); ______f.miss_hit[______r]++; ______r; })) return 13; if (__builtin_constant_p(((size <= 16 * 1024))) ? !!((size <= 16 * 1024)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/slub_def.h", .line = 184, }; ______r = !!((size <= 16 * 1024)); ______f.miss_hit[______r]++; ______r; })) return 14; if (__builtin_constant_p(((size <= 32 * 1024))) ? !!((size <= 32 * 1024)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/slub_def.h", .line = 185, }; ______r = !!((size <= 32 * 1024)); ______f.miss_hit[______r]++; ______r; })) return 15; if (__builtin_constant_p(((size <= 64 * 1024))) ? !!((size <= 64 * 1024)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/slub_def.h", .line = 186, }; ______r = !!((size <= 64 * 1024)); ______f.miss_hit[______r]++; ______r; })) return 16; if (__builtin_constant_p(((size <= 128 * 1024))) ? !!((size <= 128 * 1024)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/slub_def.h", .line = 187, }; ______r = !!((size <= 128 * 1024)); ______f.miss_hit[______r]++; ______r; })) return 17; if (__builtin_constant_p(((size <= 256 * 1024))) ? !!((size <= 256 * 1024)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/slub_def.h", .line = 188, }; ______r = !!((size <= 256 * 1024)); ______f.miss_hit[______r]++; ______r; })) return 18; if (__builtin_constant_p(((size <= 512 * 1024))) ? !!((size <= 512 * 1024)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/slub_def.h", .line = 189, }; ______r = !!((size <= 512 * 1024)); ______f.miss_hit[______r]++; ______r; })) return 19; if (__builtin_constant_p(((size <= 1024 * 1024))) ? !!((size <= 1024 * 1024)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/slub_def.h", .line = 190, }; ______r = !!((size <= 1024 * 1024)); ______f.miss_hit[______r]++; ______r; })) return 20; if (__builtin_constant_p(((size <= 2 * 1024 * 1024))) ? !!((size <= 2 * 1024 * 1024)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/slub_def.h", .line = 191, }; ______r = !!((size <= 2 * 1024 * 1024)); ______f.miss_hit[______r]++; ______r; })) return 21; do { asm volatile("1:\tud2\n" ".pushsection __bug_table,\"a\"\n" "2:\t.long 1b, %c0\n" "\t.word %c1, 0\n" "\t.org 2b+%c2\n" ".popsection" : : "i" ("include/linux/slub_def.h"), "i" (192), "i" (sizeof(struct bug_entry))); __builtin_unreachable(); } while (0); return -1; } static inline __attribute__((always_inline)) __attribute__((always_inline)) struct kmem_cache *kmalloc_slab(size_t size) { int index = kmalloc_index(size); if (__builtin_constant_p(((index == 0))) ? !!((index == 0)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/slub_def.h", .line = 214, }; ______r = !!((index == 0)); ______f.miss_hit[______r]++; ______r; })) return ((void *)0); return kmalloc_caches[index]; } void *kmem_cache_alloc(struct kmem_cache *, gfp_t); void *__kmalloc(size_t size, gfp_t flags); static inline __attribute__((always_inline)) __attribute__((always_inline)) void * kmalloc_order(size_t size, gfp_t flags, unsigned int order) { void *ret = (void *) __get_free_pages(flags | (( gfp_t)0x4000u), order); kmemleak_alloc(ret, size, 1, flags); return ret; } extern void * kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size); extern void *kmalloc_order_trace(size_t size, gfp_t flags, unsigned int order); static inline __attribute__((always_inline)) __attribute__((always_inline)) void *kmalloc_large(size_t size, gfp_t flags) { unsigned int order = get_order(size); return kmalloc_order_trace(size, flags, order); } static inline __attribute__((always_inline)) __attribute__((always_inline)) void *kmalloc(size_t size, gfp_t flags) { if (__builtin_constant_p(((__builtin_constant_p(size)))) ? !!((__builtin_constant_p(size))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/slub_def.h", .line = 257, }; ______r = !!((__builtin_constant_p(size))); ______f.miss_hit[______r]++; ______r; })) { if (__builtin_constant_p(((size > (2 * ((1UL) << 12))))) ? !!((size > (2 * ((1UL) << 12)))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/slub_def.h", .line = 258, }; ______r = !!((size > (2 * ((1UL) << 12)))); ______f.miss_hit[______r]++; ______r; })) return kmalloc_large(size, flags); if (__builtin_constant_p(((!(flags & (( gfp_t)0x01u))))) ? !!((!(flags & (( gfp_t)0x01u)))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/slub_def.h", .line = 261, }; ______r = !!((!(flags & (( gfp_t)0x01u)))); ______f.miss_hit[______r]++; ______r; })) { struct kmem_cache *s = kmalloc_slab(size); if (__builtin_constant_p(((!s))) ? !!((!s)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/slub_def.h", .line = 264, }; ______r = !!((!s)); ______f.miss_hit[______r]++; ______r; })) return ((void *)16); return kmem_cache_alloc_trace(s, flags, size); } } return __kmalloc(size, flags); } static inline __attribute__((always_inline)) void *kcalloc(size_t n, size_t size, gfp_t flags) { if (__builtin_constant_p(((size != 0 && n > (~0UL) / size))) ? !!((size != 0 && n > (~0UL) / size)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/slab.h", .line = 225, }; ______r = !!((size != 0 && n > (~0UL) / size)); ______f.miss_hit[______r]++; ______r; })) return ((void *)0); return __kmalloc(n * size, flags | (( gfp_t)0x8000u)); } static inline __attribute__((always_inline)) void *kmalloc_node(size_t size, gfp_t flags, int node) { return kmalloc(size, flags); } static inline __attribute__((always_inline)) void *__kmalloc_node(size_t size, gfp_t flags, int node) { return __kmalloc(size, flags); } void *kmem_cache_alloc(struct kmem_cache *, gfp_t); static inline __attribute__((always_inline)) void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int node) { return kmem_cache_alloc(cachep, flags); } extern void *__kmalloc_track_caller(size_t, gfp_t, unsigned long); static inline __attribute__((always_inline)) void *kmem_cache_zalloc(struct kmem_cache *k, gfp_t flags) { return kmem_cache_alloc(k, flags | (( gfp_t)0x8000u)); } static inline __attribute__((always_inline)) void *kzalloc(size_t size, gfp_t flags) { return kmalloc(size, flags | (( gfp_t)0x8000u)); } static inline __attribute__((always_inline)) void *kzalloc_node(size_t size, gfp_t flags, int node) { return kmalloc_node(size, flags | (( gfp_t)0x8000u), node); } void __attribute__ ((__section__(".init.text"))) __attribute__((__cold__)) __attribute__((no_instrument_function)) kmem_cache_init_late(void); static inline __attribute__((always_inline)) void pagefault_disable(void) { add_preempt_count(1); __asm__ __volatile__("": : :"memory"); } static inline __attribute__((always_inline)) void pagefault_enable(void) { __asm__ __volatile__("": : :"memory"); sub_preempt_count(1); __asm__ __volatile__("": : :"memory"); do { if (__builtin_constant_p((((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/uaccess.h", .line = 38, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/uaccess.h", .line = 38, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/uaccess.h", .line = 38, }; ______r = !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/uaccess.h", .line = 38, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) preempt_schedule(); } while (0); } extern long probe_kernel_read(void *dst, const void *src, size_t size); extern long __probe_kernel_read(void *dst, const void *src, size_t size); extern long __attribute__((no_instrument_function)) probe_kernel_write(void *dst, const void *src, size_t size); extern long __attribute__((no_instrument_function)) __probe_kernel_write(void *dst, const void *src, size_t size); struct scatterlist; struct crypto_ablkcipher; struct crypto_async_request; struct crypto_aead; struct crypto_blkcipher; struct crypto_hash; struct crypto_rng; struct crypto_tfm; struct crypto_type; struct aead_givcrypt_request; struct skcipher_givcrypt_request; typedef void (*crypto_completion_t)(struct crypto_async_request *req, int err); struct crypto_async_request { struct list_head list; crypto_completion_t complete; void *data; struct crypto_tfm *tfm; u32 flags; }; struct ablkcipher_request { struct crypto_async_request base; unsigned int nbytes; void *info; struct scatterlist *src; struct scatterlist *dst; void *__ctx[] __attribute__ ((__aligned__(__alignof__(unsigned long long)))); }; struct aead_request { struct crypto_async_request base; unsigned int assoclen; unsigned int cryptlen; u8 *iv; struct scatterlist *assoc; struct scatterlist *src; struct scatterlist *dst; void *__ctx[] __attribute__ ((__aligned__(__alignof__(unsigned long long)))); }; struct blkcipher_desc { struct crypto_blkcipher *tfm; void *info; u32 flags; }; struct cipher_desc { struct crypto_tfm *tfm; void (*crfn)(struct crypto_tfm *tfm, u8 *dst, const u8 *src); unsigned int (*prfn)(const struct cipher_desc *desc, u8 *dst, const u8 *src, unsigned int nbytes); void *info; }; struct hash_desc { struct crypto_hash *tfm; u32 flags; }; struct ablkcipher_alg { int (*setkey)(struct crypto_ablkcipher *tfm, const u8 *key, unsigned int keylen); int (*encrypt)(struct ablkcipher_request *req); int (*decrypt)(struct ablkcipher_request *req); int (*givencrypt)(struct skcipher_givcrypt_request *req); int (*givdecrypt)(struct skcipher_givcrypt_request *req); const char *geniv; unsigned int min_keysize; unsigned int max_keysize; unsigned int ivsize; }; struct aead_alg { int (*setkey)(struct crypto_aead *tfm, const u8 *key, unsigned int keylen); int (*setauthsize)(struct crypto_aead *tfm, unsigned int authsize); int (*encrypt)(struct aead_request *req); int (*decrypt)(struct aead_request *req); int (*givencrypt)(struct aead_givcrypt_request *req); int (*givdecrypt)(struct aead_givcrypt_request *req); const char *geniv; unsigned int ivsize; unsigned int maxauthsize; }; struct blkcipher_alg { int (*setkey)(struct crypto_tfm *tfm, const u8 *key, unsigned int keylen); int (*encrypt)(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes); int (*decrypt)(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes); const char *geniv; unsigned int min_keysize; unsigned int max_keysize; unsigned int ivsize; }; struct cipher_alg { unsigned int cia_min_keysize; unsigned int cia_max_keysize; int (*cia_setkey)(struct crypto_tfm *tfm, const u8 *key, unsigned int keylen); void (*cia_encrypt)(struct crypto_tfm *tfm, u8 *dst, const u8 *src); void (*cia_decrypt)(struct crypto_tfm *tfm, u8 *dst, const u8 *src); }; struct compress_alg { int (*coa_compress)(struct crypto_tfm *tfm, const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen); int (*coa_decompress)(struct crypto_tfm *tfm, const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen); }; struct rng_alg { int (*rng_make_random)(struct crypto_rng *tfm, u8 *rdata, unsigned int dlen); int (*rng_reset)(struct crypto_rng *tfm, u8 *seed, unsigned int slen); unsigned int seedsize; }; struct crypto_alg { struct list_head cra_list; struct list_head cra_users; u32 cra_flags; unsigned int cra_blocksize; unsigned int cra_ctxsize; unsigned int cra_alignmask; int cra_priority; atomic_t cra_refcnt; char cra_name[64]; char cra_driver_name[64]; const struct crypto_type *cra_type; union { struct ablkcipher_alg ablkcipher; struct aead_alg aead; struct blkcipher_alg blkcipher; struct cipher_alg cipher; struct compress_alg compress; struct rng_alg rng; } cra_u; int (*cra_init)(struct crypto_tfm *tfm); void (*cra_exit)(struct crypto_tfm *tfm); void (*cra_destroy)(struct crypto_alg *alg); struct module *cra_module; }; int crypto_register_alg(struct crypto_alg *alg); int crypto_unregister_alg(struct crypto_alg *alg); int crypto_has_alg(const char *name, u32 type, u32 mask); struct ablkcipher_tfm { int (*setkey)(struct crypto_ablkcipher *tfm, const u8 *key, unsigned int keylen); int (*encrypt)(struct ablkcipher_request *req); int (*decrypt)(struct ablkcipher_request *req); int (*givencrypt)(struct skcipher_givcrypt_request *req); int (*givdecrypt)(struct skcipher_givcrypt_request *req); struct crypto_ablkcipher *base; unsigned int ivsize; unsigned int reqsize; }; struct aead_tfm { int (*setkey)(struct crypto_aead *tfm, const u8 *key, unsigned int keylen); int (*encrypt)(struct aead_request *req); int (*decrypt)(struct aead_request *req); int (*givencrypt)(struct aead_givcrypt_request *req); int (*givdecrypt)(struct aead_givcrypt_request *req); struct crypto_aead *base; unsigned int ivsize; unsigned int authsize; unsigned int reqsize; }; struct blkcipher_tfm { void *iv; int (*setkey)(struct crypto_tfm *tfm, const u8 *key, unsigned int keylen); int (*encrypt)(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes); int (*decrypt)(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes); }; struct cipher_tfm { int (*cit_setkey)(struct crypto_tfm *tfm, const u8 *key, unsigned int keylen); void (*cit_encrypt_one)(struct crypto_tfm *tfm, u8 *dst, const u8 *src); void (*cit_decrypt_one)(struct crypto_tfm *tfm, u8 *dst, const u8 *src); }; struct hash_tfm { int (*init)(struct hash_desc *desc); int (*update)(struct hash_desc *desc, struct scatterlist *sg, unsigned int nsg); int (*final)(struct hash_desc *desc, u8 *out); int (*digest)(struct hash_desc *desc, struct scatterlist *sg, unsigned int nsg, u8 *out); int (*setkey)(struct crypto_hash *tfm, const u8 *key, unsigned int keylen); unsigned int digestsize; }; struct compress_tfm { int (*cot_compress)(struct crypto_tfm *tfm, const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen); int (*cot_decompress)(struct crypto_tfm *tfm, const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen); }; struct rng_tfm { int (*rng_gen_random)(struct crypto_rng *tfm, u8 *rdata, unsigned int dlen); int (*rng_reset)(struct crypto_rng *tfm, u8 *seed, unsigned int slen); }; struct crypto_tfm { u32 crt_flags; union { struct ablkcipher_tfm ablkcipher; struct aead_tfm aead; struct blkcipher_tfm blkcipher; struct cipher_tfm cipher; struct hash_tfm hash; struct compress_tfm compress; struct rng_tfm rng; } crt_u; void (*exit)(struct crypto_tfm *tfm); struct crypto_alg *__crt_alg; void *__crt_ctx[] __attribute__ ((__aligned__(__alignof__(unsigned long long)))); }; struct crypto_ablkcipher { struct crypto_tfm base; }; struct crypto_aead { struct crypto_tfm base; }; struct crypto_blkcipher { struct crypto_tfm base; }; struct crypto_cipher { struct crypto_tfm base; }; struct crypto_comp { struct crypto_tfm base; }; struct crypto_hash { struct crypto_tfm base; }; struct crypto_rng { struct crypto_tfm base; }; enum { CRYPTOA_UNSPEC, CRYPTOA_ALG, CRYPTOA_TYPE, CRYPTOA_U32, __CRYPTOA_MAX, }; struct crypto_attr_alg { char name[64]; }; struct crypto_attr_type { u32 type; u32 mask; }; struct crypto_attr_u32 { u32 num; }; struct crypto_tfm *crypto_alloc_base(const char *alg_name, u32 type, u32 mask); void crypto_destroy_tfm(void *mem, struct crypto_tfm *tfm); static inline __attribute__((always_inline)) void crypto_free_tfm(struct crypto_tfm *tfm) { return crypto_destroy_tfm(tfm, tfm); } int alg_test(const char *driver, const char *alg, u32 type, u32 mask); static inline __attribute__((always_inline)) const char *crypto_tfm_alg_name(struct crypto_tfm *tfm) { return tfm->__crt_alg->cra_name; } static inline __attribute__((always_inline)) const char *crypto_tfm_alg_driver_name(struct crypto_tfm *tfm) { return tfm->__crt_alg->cra_driver_name; } static inline __attribute__((always_inline)) int crypto_tfm_alg_priority(struct crypto_tfm *tfm) { return tfm->__crt_alg->cra_priority; } static inline __attribute__((always_inline)) const char *crypto_tfm_alg_modname(struct crypto_tfm *tfm) { return ({ struct module *__mod = (tfm->__crt_alg->cra_module); __mod ? __mod->name : "kernel"; }); } static inline __attribute__((always_inline)) u32 crypto_tfm_alg_type(struct crypto_tfm *tfm) { return tfm->__crt_alg->cra_flags & 0x0000000f; } static inline __attribute__((always_inline)) unsigned int crypto_tfm_alg_blocksize(struct crypto_tfm *tfm) { return tfm->__crt_alg->cra_blocksize; } static inline __attribute__((always_inline)) unsigned int crypto_tfm_alg_alignmask(struct crypto_tfm *tfm) { return tfm->__crt_alg->cra_alignmask; } static inline __attribute__((always_inline)) u32 crypto_tfm_get_flags(struct crypto_tfm *tfm) { return tfm->crt_flags; } static inline __attribute__((always_inline)) void crypto_tfm_set_flags(struct crypto_tfm *tfm, u32 flags) { tfm->crt_flags |= flags; } static inline __attribute__((always_inline)) void crypto_tfm_clear_flags(struct crypto_tfm *tfm, u32 flags) { tfm->crt_flags &= ~flags; } static inline __attribute__((always_inline)) void *crypto_tfm_ctx(struct crypto_tfm *tfm) { return tfm->__crt_ctx; } static inline __attribute__((always_inline)) unsigned int crypto_tfm_ctx_alignment(void) { struct crypto_tfm *tfm; return __alignof__(tfm->__crt_ctx); } static inline __attribute__((always_inline)) struct crypto_ablkcipher *__crypto_ablkcipher_cast( struct crypto_tfm *tfm) { return (struct crypto_ablkcipher *)tfm; } static inline __attribute__((always_inline)) u32 crypto_skcipher_type(u32 type) { type &= ~(0x0000000f | 0x00000200); type |= 0x00000004; return type; } static inline __attribute__((always_inline)) u32 crypto_skcipher_mask(u32 mask) { mask &= ~(0x0000000f | 0x00000200); mask |= 0x0000000c; return mask; } struct crypto_ablkcipher *crypto_alloc_ablkcipher(const char *alg_name, u32 type, u32 mask); static inline __attribute__((always_inline)) struct crypto_tfm *crypto_ablkcipher_tfm( struct crypto_ablkcipher *tfm) { return &tfm->base; } static inline __attribute__((always_inline)) void crypto_free_ablkcipher(struct crypto_ablkcipher *tfm) { crypto_free_tfm(crypto_ablkcipher_tfm(tfm)); } static inline __attribute__((always_inline)) int crypto_has_ablkcipher(const char *alg_name, u32 type, u32 mask) { return crypto_has_alg(alg_name, crypto_skcipher_type(type), crypto_skcipher_mask(mask)); } static inline __attribute__((always_inline)) struct ablkcipher_tfm *crypto_ablkcipher_crt( struct crypto_ablkcipher *tfm) { return &crypto_ablkcipher_tfm(tfm)->crt_u.ablkcipher; } static inline __attribute__((always_inline)) unsigned int crypto_ablkcipher_ivsize( struct crypto_ablkcipher *tfm) { return crypto_ablkcipher_crt(tfm)->ivsize; } static inline __attribute__((always_inline)) unsigned int crypto_ablkcipher_blocksize( struct crypto_ablkcipher *tfm) { return crypto_tfm_alg_blocksize(crypto_ablkcipher_tfm(tfm)); } static inline __attribute__((always_inline)) unsigned int crypto_ablkcipher_alignmask( struct crypto_ablkcipher *tfm) { return crypto_tfm_alg_alignmask(crypto_ablkcipher_tfm(tfm)); } static inline __attribute__((always_inline)) u32 crypto_ablkcipher_get_flags(struct crypto_ablkcipher *tfm) { return crypto_tfm_get_flags(crypto_ablkcipher_tfm(tfm)); } static inline __attribute__((always_inline)) void crypto_ablkcipher_set_flags(struct crypto_ablkcipher *tfm, u32 flags) { crypto_tfm_set_flags(crypto_ablkcipher_tfm(tfm), flags); } static inline __attribute__((always_inline)) void crypto_ablkcipher_clear_flags(struct crypto_ablkcipher *tfm, u32 flags) { crypto_tfm_clear_flags(crypto_ablkcipher_tfm(tfm), flags); } static inline __attribute__((always_inline)) int crypto_ablkcipher_setkey(struct crypto_ablkcipher *tfm, const u8 *key, unsigned int keylen) { struct ablkcipher_tfm *crt = crypto_ablkcipher_crt(tfm); return crt->setkey(crt->base, key, keylen); } static inline __attribute__((always_inline)) struct crypto_ablkcipher *crypto_ablkcipher_reqtfm( struct ablkcipher_request *req) { return __crypto_ablkcipher_cast(req->base.tfm); } static inline __attribute__((always_inline)) int crypto_ablkcipher_encrypt(struct ablkcipher_request *req) { struct ablkcipher_tfm *crt = crypto_ablkcipher_crt(crypto_ablkcipher_reqtfm(req)); return crt->encrypt(req); } static inline __attribute__((always_inline)) int crypto_ablkcipher_decrypt(struct ablkcipher_request *req) { struct ablkcipher_tfm *crt = crypto_ablkcipher_crt(crypto_ablkcipher_reqtfm(req)); return crt->decrypt(req); } static inline __attribute__((always_inline)) unsigned int crypto_ablkcipher_reqsize( struct crypto_ablkcipher *tfm) { return crypto_ablkcipher_crt(tfm)->reqsize; } static inline __attribute__((always_inline)) void ablkcipher_request_set_tfm( struct ablkcipher_request *req, struct crypto_ablkcipher *tfm) { req->base.tfm = crypto_ablkcipher_tfm(crypto_ablkcipher_crt(tfm)->base); } static inline __attribute__((always_inline)) struct ablkcipher_request *ablkcipher_request_cast( struct crypto_async_request *req) { return ({ const typeof( ((struct ablkcipher_request *)0)->base ) *__mptr = (req); (struct ablkcipher_request *)( (char *)__mptr - __builtin_offsetof(struct ablkcipher_request,base) );}); } static inline __attribute__((always_inline)) struct ablkcipher_request *ablkcipher_request_alloc( struct crypto_ablkcipher *tfm, gfp_t gfp) { struct ablkcipher_request *req; req = kmalloc(sizeof(struct ablkcipher_request) + crypto_ablkcipher_reqsize(tfm), gfp); if (__builtin_constant_p((((__builtin_constant_p(req) ? !!(req) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/crypto.h", .line = 693, }; ______r = __builtin_expect(!!(req), 1); ftrace_likely_update(&______f, ______r, 1); ______r; }))))) ? !!(((__builtin_constant_p(req) ? !!(req) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/crypto.h", .line = 693, }; ______r = __builtin_expect(!!(req), 1); ftrace_likely_update(&______f, ______r, 1); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/crypto.h", .line = 693, }; ______r = !!(((__builtin_constant_p(req) ? !!(req) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/crypto.h", .line = 693, }; ______r = __builtin_expect(!!(req), 1); ftrace_likely_update(&______f, ______r, 1); ______r; })))); ______f.miss_hit[______r]++; ______r; })) ablkcipher_request_set_tfm(req, tfm); return req; } static inline __attribute__((always_inline)) void ablkcipher_request_free(struct ablkcipher_request *req) { kzfree(req); } static inline __attribute__((always_inline)) void ablkcipher_request_set_callback( struct ablkcipher_request *req, u32 flags, crypto_completion_t complete, void *data) { req->base.complete = complete; req->base.data = data; req->base.flags = flags; } static inline __attribute__((always_inline)) void ablkcipher_request_set_crypt( struct ablkcipher_request *req, struct scatterlist *src, struct scatterlist *dst, unsigned int nbytes, void *iv) { req->src = src; req->dst = dst; req->nbytes = nbytes; req->info = iv; } static inline __attribute__((always_inline)) struct crypto_aead *__crypto_aead_cast(struct crypto_tfm *tfm) { return (struct crypto_aead *)tfm; } struct crypto_aead *crypto_alloc_aead(const char *alg_name, u32 type, u32 mask); static inline __attribute__((always_inline)) struct crypto_tfm *crypto_aead_tfm(struct crypto_aead *tfm) { return &tfm->base; } static inline __attribute__((always_inline)) void crypto_free_aead(struct crypto_aead *tfm) { crypto_free_tfm(crypto_aead_tfm(tfm)); } static inline __attribute__((always_inline)) struct aead_tfm *crypto_aead_crt(struct crypto_aead *tfm) { return &crypto_aead_tfm(tfm)->crt_u.aead; } static inline __attribute__((always_inline)) unsigned int crypto_aead_ivsize(struct crypto_aead *tfm) { return crypto_aead_crt(tfm)->ivsize; } static inline __attribute__((always_inline)) unsigned int crypto_aead_authsize(struct crypto_aead *tfm) { return crypto_aead_crt(tfm)->authsize; } static inline __attribute__((always_inline)) unsigned int crypto_aead_blocksize(struct crypto_aead *tfm) { return crypto_tfm_alg_blocksize(crypto_aead_tfm(tfm)); } static inline __attribute__((always_inline)) unsigned int crypto_aead_alignmask(struct crypto_aead *tfm) { return crypto_tfm_alg_alignmask(crypto_aead_tfm(tfm)); } static inline __attribute__((always_inline)) u32 crypto_aead_get_flags(struct crypto_aead *tfm) { return crypto_tfm_get_flags(crypto_aead_tfm(tfm)); } static inline __attribute__((always_inline)) void crypto_aead_set_flags(struct crypto_aead *tfm, u32 flags) { crypto_tfm_set_flags(crypto_aead_tfm(tfm), flags); } static inline __attribute__((always_inline)) void crypto_aead_clear_flags(struct crypto_aead *tfm, u32 flags) { crypto_tfm_clear_flags(crypto_aead_tfm(tfm), flags); } static inline __attribute__((always_inline)) int crypto_aead_setkey(struct crypto_aead *tfm, const u8 *key, unsigned int keylen) { struct aead_tfm *crt = crypto_aead_crt(tfm); return crt->setkey(crt->base, key, keylen); } int crypto_aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize); static inline __attribute__((always_inline)) struct crypto_aead *crypto_aead_reqtfm(struct aead_request *req) { return __crypto_aead_cast(req->base.tfm); } static inline __attribute__((always_inline)) int crypto_aead_encrypt(struct aead_request *req) { return crypto_aead_crt(crypto_aead_reqtfm(req))->encrypt(req); } static inline __attribute__((always_inline)) int crypto_aead_decrypt(struct aead_request *req) { return crypto_aead_crt(crypto_aead_reqtfm(req))->decrypt(req); } static inline __attribute__((always_inline)) unsigned int crypto_aead_reqsize(struct crypto_aead *tfm) { return crypto_aead_crt(tfm)->reqsize; } static inline __attribute__((always_inline)) void aead_request_set_tfm(struct aead_request *req, struct crypto_aead *tfm) { req->base.tfm = crypto_aead_tfm(crypto_aead_crt(tfm)->base); } static inline __attribute__((always_inline)) struct aead_request *aead_request_alloc(struct crypto_aead *tfm, gfp_t gfp) { struct aead_request *req; req = kmalloc(sizeof(*req) + crypto_aead_reqsize(tfm), gfp); if (__builtin_constant_p((((__builtin_constant_p(req) ? !!(req) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/crypto.h", .line = 824, }; ______r = __builtin_expect(!!(req), 1); ftrace_likely_update(&______f, ______r, 1); ______r; }))))) ? !!(((__builtin_constant_p(req) ? !!(req) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/crypto.h", .line = 824, }; ______r = __builtin_expect(!!(req), 1); ftrace_likely_update(&______f, ______r, 1); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/crypto.h", .line = 824, }; ______r = !!(((__builtin_constant_p(req) ? !!(req) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/crypto.h", .line = 824, }; ______r = __builtin_expect(!!(req), 1); ftrace_likely_update(&______f, ______r, 1); ______r; })))); ______f.miss_hit[______r]++; ______r; })) aead_request_set_tfm(req, tfm); return req; } static inline __attribute__((always_inline)) void aead_request_free(struct aead_request *req) { kzfree(req); } static inline __attribute__((always_inline)) void aead_request_set_callback(struct aead_request *req, u32 flags, crypto_completion_t complete, void *data) { req->base.complete = complete; req->base.data = data; req->base.flags = flags; } static inline __attribute__((always_inline)) void aead_request_set_crypt(struct aead_request *req, struct scatterlist *src, struct scatterlist *dst, unsigned int cryptlen, u8 *iv) { req->src = src; req->dst = dst; req->cryptlen = cryptlen; req->iv = iv; } static inline __attribute__((always_inline)) void aead_request_set_assoc(struct aead_request *req, struct scatterlist *assoc, unsigned int assoclen) { req->assoc = assoc; req->assoclen = assoclen; } static inline __attribute__((always_inline)) struct crypto_blkcipher *__crypto_blkcipher_cast( struct crypto_tfm *tfm) { return (struct crypto_blkcipher *)tfm; } static inline __attribute__((always_inline)) struct crypto_blkcipher *crypto_blkcipher_cast( struct crypto_tfm *tfm) { do { if (__builtin_constant_p((((__builtin_constant_p(crypto_tfm_alg_type(tfm) != 0x00000004) ? !!(crypto_tfm_alg_type(tfm) != 0x00000004) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/crypto.h", .line = 873, }; ______r = __builtin_expect(!!(crypto_tfm_alg_type(tfm) != 0x00000004), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(crypto_tfm_alg_type(tfm) != 0x00000004) ? !!(crypto_tfm_alg_type(tfm) != 0x00000004) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/crypto.h", .line = 873, }; ______r = __builtin_expect(!!(crypto_tfm_alg_type(tfm) != 0x00000004), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/crypto.h", .line = 873, }; ______r = !!(((__builtin_constant_p(crypto_tfm_alg_type(tfm) != 0x00000004) ? !!(crypto_tfm_alg_type(tfm) != 0x00000004) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/crypto.h", .line = 873, }; ______r = __builtin_expect(!!(crypto_tfm_alg_type(tfm) != 0x00000004), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) do { asm volatile("1:\tud2\n" ".pushsection __bug_table,\"a\"\n" "2:\t.long 1b, %c0\n" "\t.word %c1, 0\n" "\t.org 2b+%c2\n" ".popsection" : : "i" ("include/linux/crypto.h"), "i" (873), "i" (sizeof(struct bug_entry))); __builtin_unreachable(); } while (0); } while(0); return __crypto_blkcipher_cast(tfm); } static inline __attribute__((always_inline)) struct crypto_blkcipher *crypto_alloc_blkcipher( const char *alg_name, u32 type, u32 mask) { type &= ~0x0000000f; type |= 0x00000004; mask |= 0x0000000f; return __crypto_blkcipher_cast(crypto_alloc_base(alg_name, type, mask)); } static inline __attribute__((always_inline)) struct crypto_tfm *crypto_blkcipher_tfm( struct crypto_blkcipher *tfm) { return &tfm->base; } static inline __attribute__((always_inline)) void crypto_free_blkcipher(struct crypto_blkcipher *tfm) { crypto_free_tfm(crypto_blkcipher_tfm(tfm)); } static inline __attribute__((always_inline)) int crypto_has_blkcipher(const char *alg_name, u32 type, u32 mask) { type &= ~0x0000000f; type |= 0x00000004; mask |= 0x0000000f; return crypto_has_alg(alg_name, type, mask); } static inline __attribute__((always_inline)) const char *crypto_blkcipher_name(struct crypto_blkcipher *tfm) { return crypto_tfm_alg_name(crypto_blkcipher_tfm(tfm)); } static inline __attribute__((always_inline)) struct blkcipher_tfm *crypto_blkcipher_crt( struct crypto_blkcipher *tfm) { return &crypto_blkcipher_tfm(tfm)->crt_u.blkcipher; } static inline __attribute__((always_inline)) struct blkcipher_alg *crypto_blkcipher_alg( struct crypto_blkcipher *tfm) { return &crypto_blkcipher_tfm(tfm)->__crt_alg->cra_u.blkcipher; } static inline __attribute__((always_inline)) unsigned int crypto_blkcipher_ivsize(struct crypto_blkcipher *tfm) { return crypto_blkcipher_alg(tfm)->ivsize; } static inline __attribute__((always_inline)) unsigned int crypto_blkcipher_blocksize( struct crypto_blkcipher *tfm) { return crypto_tfm_alg_blocksize(crypto_blkcipher_tfm(tfm)); } static inline __attribute__((always_inline)) unsigned int crypto_blkcipher_alignmask( struct crypto_blkcipher *tfm) { return crypto_tfm_alg_alignmask(crypto_blkcipher_tfm(tfm)); } static inline __attribute__((always_inline)) u32 crypto_blkcipher_get_flags(struct crypto_blkcipher *tfm) { return crypto_tfm_get_flags(crypto_blkcipher_tfm(tfm)); } static inline __attribute__((always_inline)) void crypto_blkcipher_set_flags(struct crypto_blkcipher *tfm, u32 flags) { crypto_tfm_set_flags(crypto_blkcipher_tfm(tfm), flags); } static inline __attribute__((always_inline)) void crypto_blkcipher_clear_flags(struct crypto_blkcipher *tfm, u32 flags) { crypto_tfm_clear_flags(crypto_blkcipher_tfm(tfm), flags); } static inline __attribute__((always_inline)) int crypto_blkcipher_setkey(struct crypto_blkcipher *tfm, const u8 *key, unsigned int keylen) { return crypto_blkcipher_crt(tfm)->setkey(crypto_blkcipher_tfm(tfm), key, keylen); } static inline __attribute__((always_inline)) int crypto_blkcipher_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { desc->info = crypto_blkcipher_crt(desc->tfm)->iv; return crypto_blkcipher_crt(desc->tfm)->encrypt(desc, dst, src, nbytes); } static inline __attribute__((always_inline)) int crypto_blkcipher_encrypt_iv(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { return crypto_blkcipher_crt(desc->tfm)->encrypt(desc, dst, src, nbytes); } static inline __attribute__((always_inline)) int crypto_blkcipher_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { desc->info = crypto_blkcipher_crt(desc->tfm)->iv; return crypto_blkcipher_crt(desc->tfm)->decrypt(desc, dst, src, nbytes); } static inline __attribute__((always_inline)) int crypto_blkcipher_decrypt_iv(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { return crypto_blkcipher_crt(desc->tfm)->decrypt(desc, dst, src, nbytes); } static inline __attribute__((always_inline)) void crypto_blkcipher_set_iv(struct crypto_blkcipher *tfm, const u8 *src, unsigned int len) { __builtin_memcpy(crypto_blkcipher_crt(tfm)->iv, src, len); } static inline __attribute__((always_inline)) void crypto_blkcipher_get_iv(struct crypto_blkcipher *tfm, u8 *dst, unsigned int len) { __builtin_memcpy(dst, crypto_blkcipher_crt(tfm)->iv, len); } static inline __attribute__((always_inline)) struct crypto_cipher *__crypto_cipher_cast(struct crypto_tfm *tfm) { return (struct crypto_cipher *)tfm; } static inline __attribute__((always_inline)) struct crypto_cipher *crypto_cipher_cast(struct crypto_tfm *tfm) { do { if (__builtin_constant_p((((__builtin_constant_p(crypto_tfm_alg_type(tfm) != 0x00000001) ? !!(crypto_tfm_alg_type(tfm) != 0x00000001) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/crypto.h", .line = 1018, }; ______r = __builtin_expect(!!(crypto_tfm_alg_type(tfm) != 0x00000001), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(crypto_tfm_alg_type(tfm) != 0x00000001) ? !!(crypto_tfm_alg_type(tfm) != 0x00000001) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/crypto.h", .line = 1018, }; ______r = __builtin_expect(!!(crypto_tfm_alg_type(tfm) != 0x00000001), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/crypto.h", .line = 1018, }; ______r = !!(((__builtin_constant_p(crypto_tfm_alg_type(tfm) != 0x00000001) ? !!(crypto_tfm_alg_type(tfm) != 0x00000001) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/crypto.h", .line = 1018, }; ______r = __builtin_expect(!!(crypto_tfm_alg_type(tfm) != 0x00000001), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) do { asm volatile("1:\tud2\n" ".pushsection __bug_table,\"a\"\n" "2:\t.long 1b, %c0\n" "\t.word %c1, 0\n" "\t.org 2b+%c2\n" ".popsection" : : "i" ("include/linux/crypto.h"), "i" (1018), "i" (sizeof(struct bug_entry))); __builtin_unreachable(); } while (0); } while(0); return __crypto_cipher_cast(tfm); } static inline __attribute__((always_inline)) struct crypto_cipher *crypto_alloc_cipher(const char *alg_name, u32 type, u32 mask) { type &= ~0x0000000f; type |= 0x00000001; mask |= 0x0000000f; return __crypto_cipher_cast(crypto_alloc_base(alg_name, type, mask)); } static inline __attribute__((always_inline)) struct crypto_tfm *crypto_cipher_tfm(struct crypto_cipher *tfm) { return &tfm->base; } static inline __attribute__((always_inline)) void crypto_free_cipher(struct crypto_cipher *tfm) { crypto_free_tfm(crypto_cipher_tfm(tfm)); } static inline __attribute__((always_inline)) int crypto_has_cipher(const char *alg_name, u32 type, u32 mask) { type &= ~0x0000000f; type |= 0x00000001; mask |= 0x0000000f; return crypto_has_alg(alg_name, type, mask); } static inline __attribute__((always_inline)) struct cipher_tfm *crypto_cipher_crt(struct crypto_cipher *tfm) { return &crypto_cipher_tfm(tfm)->crt_u.cipher; } static inline __attribute__((always_inline)) unsigned int crypto_cipher_blocksize(struct crypto_cipher *tfm) { return crypto_tfm_alg_blocksize(crypto_cipher_tfm(tfm)); } static inline __attribute__((always_inline)) unsigned int crypto_cipher_alignmask(struct crypto_cipher *tfm) { return crypto_tfm_alg_alignmask(crypto_cipher_tfm(tfm)); } static inline __attribute__((always_inline)) u32 crypto_cipher_get_flags(struct crypto_cipher *tfm) { return crypto_tfm_get_flags(crypto_cipher_tfm(tfm)); } static inline __attribute__((always_inline)) void crypto_cipher_set_flags(struct crypto_cipher *tfm, u32 flags) { crypto_tfm_set_flags(crypto_cipher_tfm(tfm), flags); } static inline __attribute__((always_inline)) void crypto_cipher_clear_flags(struct crypto_cipher *tfm, u32 flags) { crypto_tfm_clear_flags(crypto_cipher_tfm(tfm), flags); } static inline __attribute__((always_inline)) int crypto_cipher_setkey(struct crypto_cipher *tfm, const u8 *key, unsigned int keylen) { return crypto_cipher_crt(tfm)->cit_setkey(crypto_cipher_tfm(tfm), key, keylen); } static inline __attribute__((always_inline)) void crypto_cipher_encrypt_one(struct crypto_cipher *tfm, u8 *dst, const u8 *src) { crypto_cipher_crt(tfm)->cit_encrypt_one(crypto_cipher_tfm(tfm), dst, src); } static inline __attribute__((always_inline)) void crypto_cipher_decrypt_one(struct crypto_cipher *tfm, u8 *dst, const u8 *src) { crypto_cipher_crt(tfm)->cit_decrypt_one(crypto_cipher_tfm(tfm), dst, src); } static inline __attribute__((always_inline)) struct crypto_hash *__crypto_hash_cast(struct crypto_tfm *tfm) { return (struct crypto_hash *)tfm; } static inline __attribute__((always_inline)) struct crypto_hash *crypto_hash_cast(struct crypto_tfm *tfm) { do { if (__builtin_constant_p((((__builtin_constant_p((crypto_tfm_alg_type(tfm) ^ 0x00000008) & 0x0000000e) ? !!((crypto_tfm_alg_type(tfm) ^ 0x00000008) & 0x0000000e) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/crypto.h", .line = 1112, }; ______r = __builtin_expect(!!((crypto_tfm_alg_type(tfm) ^ 0x00000008) & 0x0000000e), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p((crypto_tfm_alg_type(tfm) ^ 0x00000008) & 0x0000000e) ? !!((crypto_tfm_alg_type(tfm) ^ 0x00000008) & 0x0000000e) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/crypto.h", .line = 1112, }; ______r = __builtin_expect(!!((crypto_tfm_alg_type(tfm) ^ 0x00000008) & 0x0000000e), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/crypto.h" , .line = 1112 , }; ______r = !!(((__builtin_constant_p((crypto_tfm_alg_type(tfm) ^ 0x00000008) & 0x0000000e) ? !!((crypto_tfm_alg_type(tfm) ^ 0x00000008) & 0x0000000e) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/crypto.h", .line = 1112, }; ______r = __builtin_expect(!!((crypto_tfm_alg_type(tfm) ^ 0x00000008) & 0x0000000e), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) do { asm volatile("1:\tud2\n" ".pushsection __bug_table,\"a\"\n" "2:\t.long 1b, %c0\n" "\t.word %c1, 0\n" "\t.org 2b+%c2\n" ".popsection" : : "i" ( "include/linux/crypto.h" ), "i" ( 1112 ), "i" (sizeof(struct bug_entry))); __builtin_unreachable(); } while (0); } while(0) ; return __crypto_hash_cast(tfm); } static inline __attribute__((always_inline)) struct crypto_hash *crypto_alloc_hash(const char *alg_name, u32 type, u32 mask) { type &= ~0x0000000f; mask &= ~0x0000000f; type |= 0x00000008; mask |= 0x0000000e; return __crypto_hash_cast(crypto_alloc_base(alg_name, type, mask)); } static inline __attribute__((always_inline)) struct crypto_tfm *crypto_hash_tfm(struct crypto_hash *tfm) { return &tfm->base; } static inline __attribute__((always_inline)) void crypto_free_hash(struct crypto_hash *tfm) { crypto_free_tfm(crypto_hash_tfm(tfm)); } static inline __attribute__((always_inline)) int crypto_has_hash(const char *alg_name, u32 type, u32 mask) { type &= ~0x0000000f; mask &= ~0x0000000f; type |= 0x00000008; mask |= 0x0000000e; return crypto_has_alg(alg_name, type, mask); } static inline __attribute__((always_inline)) struct hash_tfm *crypto_hash_crt(struct crypto_hash *tfm) { return &crypto_hash_tfm(tfm)->crt_u.hash; } static inline __attribute__((always_inline)) unsigned int crypto_hash_blocksize(struct crypto_hash *tfm) { return crypto_tfm_alg_blocksize(crypto_hash_tfm(tfm)); } static inline __attribute__((always_inline)) unsigned int crypto_hash_alignmask(struct crypto_hash *tfm) { return crypto_tfm_alg_alignmask(crypto_hash_tfm(tfm)); } static inline __attribute__((always_inline)) unsigned int crypto_hash_digestsize(struct crypto_hash *tfm) { return crypto_hash_crt(tfm)->digestsize; } static inline __attribute__((always_inline)) u32 crypto_hash_get_flags(struct crypto_hash *tfm) { return crypto_tfm_get_flags(crypto_hash_tfm(tfm)); } static inline __attribute__((always_inline)) void crypto_hash_set_flags(struct crypto_hash *tfm, u32 flags) { crypto_tfm_set_flags(crypto_hash_tfm(tfm), flags); } static inline __attribute__((always_inline)) void crypto_hash_clear_flags(struct crypto_hash *tfm, u32 flags) { crypto_tfm_clear_flags(crypto_hash_tfm(tfm), flags); } static inline __attribute__((always_inline)) int crypto_hash_init(struct hash_desc *desc) { return crypto_hash_crt(desc->tfm)->init(desc); } static inline __attribute__((always_inline)) int crypto_hash_update(struct hash_desc *desc, struct scatterlist *sg, unsigned int nbytes) { return crypto_hash_crt(desc->tfm)->update(desc, sg, nbytes); } static inline __attribute__((always_inline)) int crypto_hash_final(struct hash_desc *desc, u8 *out) { return crypto_hash_crt(desc->tfm)->final(desc, out); } static inline __attribute__((always_inline)) int crypto_hash_digest(struct hash_desc *desc, struct scatterlist *sg, unsigned int nbytes, u8 *out) { return crypto_hash_crt(desc->tfm)->digest(desc, sg, nbytes, out); } static inline __attribute__((always_inline)) int crypto_hash_setkey(struct crypto_hash *hash, const u8 *key, unsigned int keylen) { return crypto_hash_crt(hash)->setkey(hash, key, keylen); } static inline __attribute__((always_inline)) struct crypto_comp *__crypto_comp_cast(struct crypto_tfm *tfm) { return (struct crypto_comp *)tfm; } static inline __attribute__((always_inline)) struct crypto_comp *crypto_comp_cast(struct crypto_tfm *tfm) { do { if (__builtin_constant_p((((__builtin_constant_p((crypto_tfm_alg_type(tfm) ^ 0x00000002) & 0x0000000f) ? !!((crypto_tfm_alg_type(tfm) ^ 0x00000002) & 0x0000000f) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/crypto.h", .line = 1220, }; ______r = __builtin_expect(!!((crypto_tfm_alg_type(tfm) ^ 0x00000002) & 0x0000000f), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p((crypto_tfm_alg_type(tfm) ^ 0x00000002) & 0x0000000f) ? !!((crypto_tfm_alg_type(tfm) ^ 0x00000002) & 0x0000000f) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/crypto.h", .line = 1220, }; ______r = __builtin_expect(!!((crypto_tfm_alg_type(tfm) ^ 0x00000002) & 0x0000000f), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/crypto.h" , .line = 1220 , }; ______r = !!(((__builtin_constant_p((crypto_tfm_alg_type(tfm) ^ 0x00000002) & 0x0000000f) ? !!((crypto_tfm_alg_type(tfm) ^ 0x00000002) & 0x0000000f) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/crypto.h", .line = 1220, }; ______r = __builtin_expect(!!((crypto_tfm_alg_type(tfm) ^ 0x00000002) & 0x0000000f), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) do { asm volatile("1:\tud2\n" ".pushsection __bug_table,\"a\"\n" "2:\t.long 1b, %c0\n" "\t.word %c1, 0\n" "\t.org 2b+%c2\n" ".popsection" : : "i" ( "include/linux/crypto.h" ), "i" ( 1220 ), "i" (sizeof(struct bug_entry))); __builtin_unreachable(); } while (0); } while(0) ; return __crypto_comp_cast(tfm); } static inline __attribute__((always_inline)) struct crypto_comp *crypto_alloc_comp(const char *alg_name, u32 type, u32 mask) { type &= ~0x0000000f; type |= 0x00000002; mask |= 0x0000000f; return __crypto_comp_cast(crypto_alloc_base(alg_name, type, mask)); } static inline __attribute__((always_inline)) struct crypto_tfm *crypto_comp_tfm(struct crypto_comp *tfm) { return &tfm->base; } static inline __attribute__((always_inline)) void crypto_free_comp(struct crypto_comp *tfm) { crypto_free_tfm(crypto_comp_tfm(tfm)); } static inline __attribute__((always_inline)) int crypto_has_comp(const char *alg_name, u32 type, u32 mask) { type &= ~0x0000000f; type |= 0x00000002; mask |= 0x0000000f; return crypto_has_alg(alg_name, type, mask); } static inline __attribute__((always_inline)) const char *crypto_comp_name(struct crypto_comp *tfm) { return crypto_tfm_alg_name(crypto_comp_tfm(tfm)); } static inline __attribute__((always_inline)) struct compress_tfm *crypto_comp_crt(struct crypto_comp *tfm) { return &crypto_comp_tfm(tfm)->crt_u.compress; } static inline __attribute__((always_inline)) int crypto_comp_compress(struct crypto_comp *tfm, const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen) { return crypto_comp_crt(tfm)->cot_compress(crypto_comp_tfm(tfm), src, slen, dst, dlen); } static inline __attribute__((always_inline)) int crypto_comp_decompress(struct crypto_comp *tfm, const u8 *src, unsigned int slen, u8 *dst, unsigned int *dlen) { return crypto_comp_crt(tfm)->cot_decompress(crypto_comp_tfm(tfm), src, slen, dst, dlen); } struct module; struct rtattr; struct seq_file; struct crypto_type { unsigned int (*ctxsize)(struct crypto_alg *alg, u32 type, u32 mask); unsigned int (*extsize)(struct crypto_alg *alg); int (*init)(struct crypto_tfm *tfm, u32 type, u32 mask); int (*init_tfm)(struct crypto_tfm *tfm); void (*show)(struct seq_file *m, struct crypto_alg *alg); struct crypto_alg *(*lookup)(const char *name, u32 type, u32 mask); unsigned int type; unsigned int maskclear; unsigned int maskset; unsigned int tfmsize; }; struct crypto_instance { struct crypto_alg alg; struct crypto_template *tmpl; struct hlist_node list; void *__ctx[] __attribute__ ((__aligned__(__alignof__(unsigned long long)))); }; struct crypto_template { struct list_head list; struct hlist_head instances; struct module *module; struct crypto_instance *(*alloc)(struct rtattr **tb); void (*free)(struct crypto_instance *inst); int (*create)(struct crypto_template *tmpl, struct rtattr **tb); char name[64]; }; struct crypto_spawn { struct list_head list; struct crypto_alg *alg; struct crypto_instance *inst; const struct crypto_type *frontend; u32 mask; }; struct crypto_queue { struct list_head list; struct list_head *backlog; unsigned int qlen; unsigned int max_qlen; }; struct scatter_walk { struct scatterlist *sg; unsigned int offset; }; struct blkcipher_walk { union { struct { struct page *page; unsigned long offset; } phys; struct { u8 *page; u8 *addr; } virt; } src, dst; struct scatter_walk in; unsigned int nbytes; struct scatter_walk out; unsigned int total; void *page; u8 *buffer; u8 *iv; int flags; unsigned int blocksize; }; struct ablkcipher_walk { struct { struct page *page; unsigned int offset; } src, dst; struct scatter_walk in; unsigned int nbytes; struct scatter_walk out; unsigned int total; struct list_head buffers; u8 *iv_buffer; u8 *iv; int flags; unsigned int blocksize; }; extern const struct crypto_type crypto_ablkcipher_type; extern const struct crypto_type crypto_aead_type; extern const struct crypto_type crypto_blkcipher_type; void crypto_mod_put(struct crypto_alg *alg); int crypto_register_template(struct crypto_template *tmpl); void crypto_unregister_template(struct crypto_template *tmpl); struct crypto_template *crypto_lookup_template(const char *name); int crypto_register_instance(struct crypto_template *tmpl, struct crypto_instance *inst); int crypto_init_spawn(struct crypto_spawn *spawn, struct crypto_alg *alg, struct crypto_instance *inst, u32 mask); int crypto_init_spawn2(struct crypto_spawn *spawn, struct crypto_alg *alg, struct crypto_instance *inst, const struct crypto_type *frontend); void crypto_drop_spawn(struct crypto_spawn *spawn); struct crypto_tfm *crypto_spawn_tfm(struct crypto_spawn *spawn, u32 type, u32 mask); void *crypto_spawn_tfm2(struct crypto_spawn *spawn); static inline __attribute__((always_inline)) void crypto_set_spawn(struct crypto_spawn *spawn, struct crypto_instance *inst) { spawn->inst = inst; } struct crypto_attr_type *crypto_get_attr_type(struct rtattr **tb); int crypto_check_attr_type(struct rtattr **tb, u32 type); const char *crypto_attr_alg_name(struct rtattr *rta); struct crypto_alg *crypto_attr_alg2(struct rtattr *rta, const struct crypto_type *frontend, u32 type, u32 mask); static inline __attribute__((always_inline)) struct crypto_alg *crypto_attr_alg(struct rtattr *rta, u32 type, u32 mask) { return crypto_attr_alg2(rta, ((void *)0), type, mask); } int crypto_attr_u32(struct rtattr *rta, u32 *num); void *crypto_alloc_instance2(const char *name, struct crypto_alg *alg, unsigned int head); struct crypto_instance *crypto_alloc_instance(const char *name, struct crypto_alg *alg); void crypto_init_queue(struct crypto_queue *queue, unsigned int max_qlen); int crypto_enqueue_request(struct crypto_queue *queue, struct crypto_async_request *request); void *__crypto_dequeue_request(struct crypto_queue *queue, unsigned int offset); struct crypto_async_request *crypto_dequeue_request(struct crypto_queue *queue); int crypto_tfm_in_queue(struct crypto_queue *queue, struct crypto_tfm *tfm); void crypto_inc(u8 *a, unsigned int size); void crypto_xor(u8 *dst, const u8 *src, unsigned int size); int blkcipher_walk_done(struct blkcipher_desc *desc, struct blkcipher_walk *walk, int err); int blkcipher_walk_virt(struct blkcipher_desc *desc, struct blkcipher_walk *walk); int blkcipher_walk_phys(struct blkcipher_desc *desc, struct blkcipher_walk *walk); int blkcipher_walk_virt_block(struct blkcipher_desc *desc, struct blkcipher_walk *walk, unsigned int blocksize); int ablkcipher_walk_done(struct ablkcipher_request *req, struct ablkcipher_walk *walk, int err); int ablkcipher_walk_phys(struct ablkcipher_request *req, struct ablkcipher_walk *walk); void __ablkcipher_walk_complete(struct ablkcipher_walk *walk); static inline __attribute__((always_inline)) void *crypto_tfm_ctx_aligned(struct crypto_tfm *tfm) { return ((typeof(crypto_tfm_ctx(tfm)))(((((unsigned long)(crypto_tfm_ctx(tfm)))) + ((typeof(((unsigned long)(crypto_tfm_ctx(tfm)))))(((crypto_tfm_alg_alignmask(tfm) + 1))) - 1)) & ~((typeof(((unsigned long)(crypto_tfm_ctx(tfm)))))(((crypto_tfm_alg_alignmask(tfm) + 1))) - 1))) ; } static inline __attribute__((always_inline)) struct crypto_instance *crypto_tfm_alg_instance( struct crypto_tfm *tfm) { return ({ const typeof( ((struct crypto_instance *)0)->alg ) *__mptr = (tfm->__crt_alg); (struct crypto_instance *)( (char *)__mptr - __builtin_offsetof(struct crypto_instance,alg) );}); } static inline __attribute__((always_inline)) void *crypto_instance_ctx(struct crypto_instance *inst) { return inst->__ctx; } static inline __attribute__((always_inline)) struct ablkcipher_alg *crypto_ablkcipher_alg( struct crypto_ablkcipher *tfm) { return &crypto_ablkcipher_tfm(tfm)->__crt_alg->cra_u.ablkcipher; } static inline __attribute__((always_inline)) void *crypto_ablkcipher_ctx(struct crypto_ablkcipher *tfm) { return crypto_tfm_ctx(&tfm->base); } static inline __attribute__((always_inline)) void *crypto_ablkcipher_ctx_aligned(struct crypto_ablkcipher *tfm) { return crypto_tfm_ctx_aligned(&tfm->base); } static inline __attribute__((always_inline)) struct aead_alg *crypto_aead_alg(struct crypto_aead *tfm) { return &crypto_aead_tfm(tfm)->__crt_alg->cra_u.aead; } static inline __attribute__((always_inline)) void *crypto_aead_ctx(struct crypto_aead *tfm) { return crypto_tfm_ctx(&tfm->base); } static inline __attribute__((always_inline)) struct crypto_instance *crypto_aead_alg_instance( struct crypto_aead *aead) { return crypto_tfm_alg_instance(&aead->base); } static inline __attribute__((always_inline)) struct crypto_blkcipher *crypto_spawn_blkcipher( struct crypto_spawn *spawn) { u32 type = 0x00000004; u32 mask = 0x0000000f; return __crypto_blkcipher_cast(crypto_spawn_tfm(spawn, type, mask)); } static inline __attribute__((always_inline)) void *crypto_blkcipher_ctx(struct crypto_blkcipher *tfm) { return crypto_tfm_ctx(&tfm->base); } static inline __attribute__((always_inline)) void *crypto_blkcipher_ctx_aligned(struct crypto_blkcipher *tfm) { return crypto_tfm_ctx_aligned(&tfm->base); } static inline __attribute__((always_inline)) struct crypto_cipher *crypto_spawn_cipher( struct crypto_spawn *spawn) { u32 type = 0x00000001; u32 mask = 0x0000000f; return __crypto_cipher_cast(crypto_spawn_tfm(spawn, type, mask)); } static inline __attribute__((always_inline)) struct cipher_alg *crypto_cipher_alg(struct crypto_cipher *tfm) { return &crypto_cipher_tfm(tfm)->__crt_alg->cra_u.cipher; } static inline __attribute__((always_inline)) struct crypto_hash *crypto_spawn_hash(struct crypto_spawn *spawn) { u32 type = 0x00000008; u32 mask = 0x0000000e; return __crypto_hash_cast(crypto_spawn_tfm(spawn, type, mask)); } static inline __attribute__((always_inline)) void *crypto_hash_ctx(struct crypto_hash *tfm) { return crypto_tfm_ctx(&tfm->base); } static inline __attribute__((always_inline)) void *crypto_hash_ctx_aligned(struct crypto_hash *tfm) { return crypto_tfm_ctx_aligned(&tfm->base); } static inline __attribute__((always_inline)) void blkcipher_walk_init(struct blkcipher_walk *walk, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { walk->in.sg = src; walk->out.sg = dst; walk->total = nbytes; } static inline __attribute__((always_inline)) void ablkcipher_walk_init(struct ablkcipher_walk *walk, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { walk->in.sg = src; walk->out.sg = dst; walk->total = nbytes; INIT_LIST_HEAD(&walk->buffers); } static inline __attribute__((always_inline)) void ablkcipher_walk_complete(struct ablkcipher_walk *walk) { if (__builtin_constant_p((((__builtin_constant_p(!list_empty(&walk->buffers)) ? !!(!list_empty(&walk->buffers)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/crypto/algapi.h", .line = 322, }; ______r = __builtin_expect(!!(!list_empty(&walk->buffers)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(!list_empty(&walk->buffers)) ? !!(!list_empty(&walk->buffers)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/crypto/algapi.h", .line = 322, }; ______r = __builtin_expect(!!(!list_empty(&walk->buffers)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/crypto/algapi.h", .line = 322, }; ______r = !!(((__builtin_constant_p(!list_empty(&walk->buffers)) ? !!(!list_empty(&walk->buffers)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/crypto/algapi.h", .line = 322, }; ______r = __builtin_expect(!!(!list_empty(&walk->buffers)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) __ablkcipher_walk_complete(walk); } static inline __attribute__((always_inline)) struct crypto_async_request *crypto_get_backlog( struct crypto_queue *queue) { return queue->backlog == &queue->list ? ((void *)0) : ({ const typeof( ((struct crypto_async_request *)0)->list ) *__mptr = (queue->backlog); (struct crypto_async_request *)( (char *)__mptr - __builtin_offsetof(struct crypto_async_request,list) );}); } static inline __attribute__((always_inline)) int ablkcipher_enqueue_request(struct crypto_queue *queue, struct ablkcipher_request *request) { return crypto_enqueue_request(queue, &request->base); } static inline __attribute__((always_inline)) struct ablkcipher_request *ablkcipher_dequeue_request( struct crypto_queue *queue) { return ablkcipher_request_cast(crypto_dequeue_request(queue)); } static inline __attribute__((always_inline)) void *ablkcipher_request_ctx(struct ablkcipher_request *req) { return req->__ctx; } static inline __attribute__((always_inline)) int ablkcipher_tfm_in_queue(struct crypto_queue *queue, struct crypto_ablkcipher *tfm) { return crypto_tfm_in_queue(queue, crypto_ablkcipher_tfm(tfm)); } static inline __attribute__((always_inline)) void *aead_request_ctx(struct aead_request *req) { return req->__ctx; } static inline __attribute__((always_inline)) void aead_request_complete(struct aead_request *req, int err) { req->base.complete(&req->base, err); } static inline __attribute__((always_inline)) u32 aead_request_flags(struct aead_request *req) { return req->base.flags; } static inline __attribute__((always_inline)) struct crypto_alg *crypto_get_attr_alg(struct rtattr **tb, u32 type, u32 mask) { return crypto_attr_alg(tb[1], type, mask); } static inline __attribute__((always_inline)) int crypto_requires_sync(u32 type, u32 mask) { return (type ^ 0x00000080) & mask & 0x00000080; } struct crypto_aes_ctx { u32 key_enc[((15 * 16) / sizeof(u32))]; u32 key_dec[((15 * 16) / sizeof(u32))]; u32 key_length; }; extern const u32 crypto_ft_tab[4][256]; extern const u32 crypto_fl_tab[4][256]; extern const u32 crypto_it_tab[4][256]; extern const u32 crypto_il_tab[4][256]; int crypto_aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, unsigned int key_len); int crypto_aes_expand_key(struct crypto_aes_ctx *ctx, const u8 *in_key, unsigned int key_len); struct crypto_ahash; struct hash_alg_common { unsigned int digestsize; unsigned int statesize; struct crypto_alg base; }; struct ahash_request { struct crypto_async_request base; unsigned int nbytes; struct scatterlist *src; u8 *result; void *priv; void *__ctx[] __attribute__ ((__aligned__(__alignof__(unsigned long long)))); }; struct ahash_alg { int (*init)(struct ahash_request *req); int (*update)(struct ahash_request *req); int (*final)(struct ahash_request *req); int (*finup)(struct ahash_request *req); int (*digest)(struct ahash_request *req); int (*export)(struct ahash_request *req, void *out); int (*import)(struct ahash_request *req, const void *in); int (*setkey)(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen); struct hash_alg_common halg; }; struct shash_desc { struct crypto_shash *tfm; u32 flags; void *__ctx[] __attribute__ ((__aligned__(__alignof__(unsigned long long)))); }; struct shash_alg { int (*init)(struct shash_desc *desc); int (*update)(struct shash_desc *desc, const u8 *data, unsigned int len); int (*final)(struct shash_desc *desc, u8 *out); int (*finup)(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out); int (*digest)(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out); int (*export)(struct shash_desc *desc, void *out); int (*import)(struct shash_desc *desc, const void *in); int (*setkey)(struct crypto_shash *tfm, const u8 *key, unsigned int keylen); unsigned int descsize; unsigned int digestsize __attribute__ ((aligned(__alignof__(struct hash_alg_common)))); unsigned int statesize; struct crypto_alg base; }; struct crypto_ahash { int (*init)(struct ahash_request *req); int (*update)(struct ahash_request *req); int (*final)(struct ahash_request *req); int (*finup)(struct ahash_request *req); int (*digest)(struct ahash_request *req); int (*export)(struct ahash_request *req, void *out); int (*import)(struct ahash_request *req, const void *in); int (*setkey)(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen); unsigned int reqsize; struct crypto_tfm base; }; struct crypto_shash { unsigned int descsize; struct crypto_tfm base; }; static inline __attribute__((always_inline)) struct crypto_ahash *__crypto_ahash_cast(struct crypto_tfm *tfm) { return ({ const typeof( ((struct crypto_ahash *)0)->base ) *__mptr = (tfm); (struct crypto_ahash *)( (char *)__mptr - __builtin_offsetof(struct crypto_ahash,base) );}); } struct crypto_ahash *crypto_alloc_ahash(const char *alg_name, u32 type, u32 mask); static inline __attribute__((always_inline)) struct crypto_tfm *crypto_ahash_tfm(struct crypto_ahash *tfm) { return &tfm->base; } static inline __attribute__((always_inline)) void crypto_free_ahash(struct crypto_ahash *tfm) { crypto_destroy_tfm(tfm, crypto_ahash_tfm(tfm)); } static inline __attribute__((always_inline)) unsigned int crypto_ahash_alignmask( struct crypto_ahash *tfm) { return crypto_tfm_alg_alignmask(crypto_ahash_tfm(tfm)); } static inline __attribute__((always_inline)) struct hash_alg_common *__crypto_hash_alg_common( struct crypto_alg *alg) { return ({ const typeof( ((struct hash_alg_common *)0)->base ) *__mptr = (alg); (struct hash_alg_common *)( (char *)__mptr - __builtin_offsetof(struct hash_alg_common,base) );}); } static inline __attribute__((always_inline)) struct hash_alg_common *crypto_hash_alg_common( struct crypto_ahash *tfm) { return __crypto_hash_alg_common(crypto_ahash_tfm(tfm)->__crt_alg); } static inline __attribute__((always_inline)) unsigned int crypto_ahash_digestsize(struct crypto_ahash *tfm) { return crypto_hash_alg_common(tfm)->digestsize; } static inline __attribute__((always_inline)) unsigned int crypto_ahash_statesize(struct crypto_ahash *tfm) { return crypto_hash_alg_common(tfm)->statesize; } static inline __attribute__((always_inline)) u32 crypto_ahash_get_flags(struct crypto_ahash *tfm) { return crypto_tfm_get_flags(crypto_ahash_tfm(tfm)); } static inline __attribute__((always_inline)) void crypto_ahash_set_flags(struct crypto_ahash *tfm, u32 flags) { crypto_tfm_set_flags(crypto_ahash_tfm(tfm), flags); } static inline __attribute__((always_inline)) void crypto_ahash_clear_flags(struct crypto_ahash *tfm, u32 flags) { crypto_tfm_clear_flags(crypto_ahash_tfm(tfm), flags); } static inline __attribute__((always_inline)) struct crypto_ahash *crypto_ahash_reqtfm( struct ahash_request *req) { return __crypto_ahash_cast(req->base.tfm); } static inline __attribute__((always_inline)) unsigned int crypto_ahash_reqsize(struct crypto_ahash *tfm) { return tfm->reqsize; } static inline __attribute__((always_inline)) void *ahash_request_ctx(struct ahash_request *req) { return req->__ctx; } int crypto_ahash_setkey(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen); int crypto_ahash_finup(struct ahash_request *req); int crypto_ahash_final(struct ahash_request *req); int crypto_ahash_digest(struct ahash_request *req); static inline __attribute__((always_inline)) int crypto_ahash_export(struct ahash_request *req, void *out) { return crypto_ahash_reqtfm(req)->export(req, out); } static inline __attribute__((always_inline)) int crypto_ahash_import(struct ahash_request *req, const void *in) { return crypto_ahash_reqtfm(req)->import(req, in); } static inline __attribute__((always_inline)) int crypto_ahash_init(struct ahash_request *req) { return crypto_ahash_reqtfm(req)->init(req); } static inline __attribute__((always_inline)) int crypto_ahash_update(struct ahash_request *req) { return crypto_ahash_reqtfm(req)->update(req); } static inline __attribute__((always_inline)) void ahash_request_set_tfm(struct ahash_request *req, struct crypto_ahash *tfm) { req->base.tfm = crypto_ahash_tfm(tfm); } static inline __attribute__((always_inline)) struct ahash_request *ahash_request_alloc( struct crypto_ahash *tfm, gfp_t gfp) { struct ahash_request *req; req = kmalloc(sizeof(struct ahash_request) + crypto_ahash_reqsize(tfm), gfp); if (__builtin_constant_p((((__builtin_constant_p(req) ? !!(req) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/crypto/hash.h", .line = 222, }; ______r = __builtin_expect(!!(req), 1); ftrace_likely_update(&______f, ______r, 1); ______r; }))))) ? !!(((__builtin_constant_p(req) ? !!(req) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/crypto/hash.h", .line = 222, }; ______r = __builtin_expect(!!(req), 1); ftrace_likely_update(&______f, ______r, 1); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/crypto/hash.h", .line = 222, }; ______r = !!(((__builtin_constant_p(req) ? !!(req) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/crypto/hash.h", .line = 222, }; ______r = __builtin_expect(!!(req), 1); ftrace_likely_update(&______f, ______r, 1); ______r; })))); ______f.miss_hit[______r]++; ______r; })) ahash_request_set_tfm(req, tfm); return req; } static inline __attribute__((always_inline)) void ahash_request_free(struct ahash_request *req) { kzfree(req); } static inline __attribute__((always_inline)) struct ahash_request *ahash_request_cast( struct crypto_async_request *req) { return ({ const typeof( ((struct ahash_request *)0)->base ) *__mptr = (req); (struct ahash_request *)( (char *)__mptr - __builtin_offsetof(struct ahash_request,base) );}); } static inline __attribute__((always_inline)) void ahash_request_set_callback(struct ahash_request *req, u32 flags, crypto_completion_t complete, void *data) { req->base.complete = complete; req->base.data = data; req->base.flags = flags; } static inline __attribute__((always_inline)) void ahash_request_set_crypt(struct ahash_request *req, struct scatterlist *src, u8 *result, unsigned int nbytes) { req->src = src; req->nbytes = nbytes; req->result = result; } struct crypto_shash *crypto_alloc_shash(const char *alg_name, u32 type, u32 mask); static inline __attribute__((always_inline)) struct crypto_tfm *crypto_shash_tfm(struct crypto_shash *tfm) { return &tfm->base; } static inline __attribute__((always_inline)) void crypto_free_shash(struct crypto_shash *tfm) { crypto_destroy_tfm(tfm, crypto_shash_tfm(tfm)); } static inline __attribute__((always_inline)) unsigned int crypto_shash_alignmask( struct crypto_shash *tfm) { return crypto_tfm_alg_alignmask(crypto_shash_tfm(tfm)); } static inline __attribute__((always_inline)) unsigned int crypto_shash_blocksize(struct crypto_shash *tfm) { return crypto_tfm_alg_blocksize(crypto_shash_tfm(tfm)); } static inline __attribute__((always_inline)) struct shash_alg *__crypto_shash_alg(struct crypto_alg *alg) { return ({ const typeof( ((struct shash_alg *)0)->base ) *__mptr = (alg); (struct shash_alg *)( (char *)__mptr - __builtin_offsetof(struct shash_alg,base) );}); } static inline __attribute__((always_inline)) struct shash_alg *crypto_shash_alg(struct crypto_shash *tfm) { return __crypto_shash_alg(crypto_shash_tfm(tfm)->__crt_alg); } static inline __attribute__((always_inline)) unsigned int crypto_shash_digestsize(struct crypto_shash *tfm) { return crypto_shash_alg(tfm)->digestsize; } static inline __attribute__((always_inline)) unsigned int crypto_shash_statesize(struct crypto_shash *tfm) { return crypto_shash_alg(tfm)->statesize; } static inline __attribute__((always_inline)) u32 crypto_shash_get_flags(struct crypto_shash *tfm) { return crypto_tfm_get_flags(crypto_shash_tfm(tfm)); } static inline __attribute__((always_inline)) void crypto_shash_set_flags(struct crypto_shash *tfm, u32 flags) { crypto_tfm_set_flags(crypto_shash_tfm(tfm), flags); } static inline __attribute__((always_inline)) void crypto_shash_clear_flags(struct crypto_shash *tfm, u32 flags) { crypto_tfm_clear_flags(crypto_shash_tfm(tfm), flags); } static inline __attribute__((always_inline)) unsigned int crypto_shash_descsize(struct crypto_shash *tfm) { return tfm->descsize; } static inline __attribute__((always_inline)) void *shash_desc_ctx(struct shash_desc *desc) { return desc->__ctx; } int crypto_shash_setkey(struct crypto_shash *tfm, const u8 *key, unsigned int keylen); int crypto_shash_digest(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out); static inline __attribute__((always_inline)) int crypto_shash_export(struct shash_desc *desc, void *out) { return crypto_shash_alg(desc->tfm)->export(desc, out); } static inline __attribute__((always_inline)) int crypto_shash_import(struct shash_desc *desc, const void *in) { return crypto_shash_alg(desc->tfm)->import(desc, in); } static inline __attribute__((always_inline)) int crypto_shash_init(struct shash_desc *desc) { return crypto_shash_alg(desc->tfm)->init(desc); } int crypto_shash_update(struct shash_desc *desc, const u8 *data, unsigned int len); int crypto_shash_final(struct shash_desc *desc, u8 *out); int crypto_shash_finup(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out); struct cryptd_ablkcipher { struct crypto_ablkcipher base; }; static inline __attribute__((always_inline)) struct cryptd_ablkcipher *__cryptd_ablkcipher_cast( struct crypto_ablkcipher *tfm) { return (struct cryptd_ablkcipher *)tfm; } struct cryptd_ablkcipher *cryptd_alloc_ablkcipher(const char *alg_name, u32 type, u32 mask); struct crypto_blkcipher *cryptd_ablkcipher_child(struct cryptd_ablkcipher *tfm); void cryptd_free_ablkcipher(struct cryptd_ablkcipher *tfm); struct cryptd_ahash { struct crypto_ahash base; }; static inline __attribute__((always_inline)) struct cryptd_ahash *__cryptd_ahash_cast( struct crypto_ahash *tfm) { return (struct cryptd_ahash *)tfm; } struct cryptd_ahash *cryptd_alloc_ahash(const char *alg_name, u32 type, u32 mask); struct crypto_shash *cryptd_ahash_child(struct cryptd_ahash *tfm); struct shash_desc *cryptd_shash_desc(struct ahash_request *req); void cryptd_free_ahash(struct cryptd_ahash *tfm); struct cryptd_aead { struct crypto_aead base; }; static inline __attribute__((always_inline)) struct cryptd_aead *__cryptd_aead_cast( struct crypto_aead *tfm) { return (struct cryptd_aead *)tfm; } struct cryptd_aead *cryptd_alloc_aead(const char *alg_name, u32 type, u32 mask); struct crypto_aead *cryptd_aead_child(struct cryptd_aead *tfm); void cryptd_free_aead(struct cryptd_aead *tfm); struct sched_param { int sched_priority; }; struct task_struct; typedef struct __user_cap_header_struct { __u32 version; int pid; } *cap_user_header_t; typedef struct __user_cap_data_struct { __u32 effective; __u32 permitted; __u32 inheritable; } *cap_user_data_t; struct vfs_cap_data { __le32 magic_etc; struct { __le32 permitted; __le32 inheritable; } data[2]; }; extern int file_caps_enabled; typedef struct kernel_cap_struct { __u32 cap[2]; } kernel_cap_t; struct cpu_vfs_cap_data { __u32 magic_etc; kernel_cap_t permitted; kernel_cap_t inheritable; }; struct dentry; struct user_namespace; struct user_namespace *current_user_ns(void); extern const kernel_cap_t __cap_empty_set; extern const kernel_cap_t __cap_full_set; extern const kernel_cap_t __cap_init_eff_set; static inline __attribute__((always_inline)) kernel_cap_t cap_combine(const kernel_cap_t a, const kernel_cap_t b) { kernel_cap_t dest; do { unsigned __capi; for (__capi = 0; __capi < 2; ++__capi) { dest.cap[__capi] = a.cap[__capi] | b.cap[__capi]; } } while (0); return dest; } static inline __attribute__((always_inline)) kernel_cap_t cap_intersect(const kernel_cap_t a, const kernel_cap_t b) { kernel_cap_t dest; do { unsigned __capi; for (__capi = 0; __capi < 2; ++__capi) { dest.cap[__capi] = a.cap[__capi] & b.cap[__capi]; } } while (0); return dest; } static inline __attribute__((always_inline)) kernel_cap_t cap_drop(const kernel_cap_t a, const kernel_cap_t drop) { kernel_cap_t dest; do { unsigned __capi; for (__capi = 0; __capi < 2; ++__capi) { dest.cap[__capi] = a.cap[__capi] &~ drop.cap[__capi]; } } while (0); return dest; } static inline __attribute__((always_inline)) kernel_cap_t cap_invert(const kernel_cap_t c) { kernel_cap_t dest; do { unsigned __capi; for (__capi = 0; __capi < 2; ++__capi) { dest.cap[__capi] = ~ c.cap[__capi]; } } while (0); return dest; } static inline __attribute__((always_inline)) int cap_isclear(const kernel_cap_t a) { unsigned __capi; for (__capi = 0; __capi < 2; ++__capi) { if (__builtin_constant_p(((a.cap[__capi] != 0))) ? !!((a.cap[__capi] != 0)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/capability.h", .line = 486, }; ______r = !!((a.cap[__capi] != 0)); ______f.miss_hit[______r]++; ______r; })) return 0; } return 1; } static inline __attribute__((always_inline)) int cap_issubset(const kernel_cap_t a, const kernel_cap_t set) { kernel_cap_t dest; dest = cap_drop(a, set); return cap_isclear(dest); } static inline __attribute__((always_inline)) int cap_is_fs_cap(int cap) { const kernel_cap_t __cap_fs_set = ((kernel_cap_t){{ ((1 << ((0) & 31)) | (1 << ((27) & 31)) | (1 << ((1) & 31)) | (1 << ((2) & 31)) | (1 << ((3) & 31)) | (1 << ((4) & 31))) | (1 << ((9) & 31)), ((1 << ((32) & 31))) } }); return !!((1 << ((cap) & 31)) & __cap_fs_set.cap[((cap) >> 5)]); } static inline __attribute__((always_inline)) kernel_cap_t cap_drop_fs_set(const kernel_cap_t a) { const kernel_cap_t __cap_fs_set = ((kernel_cap_t){{ ((1 << ((0) & 31)) | (1 << ((27) & 31)) | (1 << ((1) & 31)) | (1 << ((2) & 31)) | (1 << ((3) & 31)) | (1 << ((4) & 31))) | (1 << ((9) & 31)), ((1 << ((32) & 31))) } }); return cap_drop(a, __cap_fs_set); } static inline __attribute__((always_inline)) kernel_cap_t cap_raise_fs_set(const kernel_cap_t a, const kernel_cap_t permitted) { const kernel_cap_t __cap_fs_set = ((kernel_cap_t){{ ((1 << ((0) & 31)) | (1 << ((27) & 31)) | (1 << ((1) & 31)) | (1 << ((2) & 31)) | (1 << ((3) & 31)) | (1 << ((4) & 31))) | (1 << ((9) & 31)), ((1 << ((32) & 31))) } }); return cap_combine(a, cap_intersect(permitted, __cap_fs_set)); } static inline __attribute__((always_inline)) kernel_cap_t cap_drop_nfsd_set(const kernel_cap_t a) { const kernel_cap_t __cap_fs_set = ((kernel_cap_t){{ ((1 << ((0) & 31)) | (1 << ((27) & 31)) | (1 << ((1) & 31)) | (1 << ((2) & 31)) | (1 << ((3) & 31)) | (1 << ((4) & 31))) | (1 << ((24) & 31)), ((1 << ((32) & 31))) } }); return cap_drop(a, __cap_fs_set); } static inline __attribute__((always_inline)) kernel_cap_t cap_raise_nfsd_set(const kernel_cap_t a, const kernel_cap_t permitted) { const kernel_cap_t __cap_nfsd_set = ((kernel_cap_t){{ ((1 << ((0) & 31)) | (1 << ((27) & 31)) | (1 << ((1) & 31)) | (1 << ((2) & 31)) | (1 << ((3) & 31)) | (1 << ((4) & 31))) | (1 << ((24) & 31)), ((1 << ((32) & 31))) } }); return cap_combine(a, cap_intersect(permitted, __cap_nfsd_set)); } extern bool has_capability(struct task_struct *t, int cap); extern bool has_ns_capability(struct task_struct *t, struct user_namespace *ns, int cap); extern bool has_capability_noaudit(struct task_struct *t, int cap); extern bool capable(int cap); extern bool ns_capable(struct user_namespace *ns, int cap); extern bool task_ns_capable(struct task_struct *t, int cap); extern bool nsown_capable(int cap); extern int get_vfs_caps_from_disk(const struct dentry *dentry, struct cpu_vfs_cap_data *cpu_caps); struct rb_node { unsigned long rb_parent_color; struct rb_node *rb_right; struct rb_node *rb_left; } __attribute__((aligned(sizeof(long)))); struct rb_root { struct rb_node *rb_node; }; static inline __attribute__((always_inline)) void rb_set_parent(struct rb_node *rb, struct rb_node *p) { rb->rb_parent_color = (rb->rb_parent_color & 3) | (unsigned long)p; } static inline __attribute__((always_inline)) void rb_set_color(struct rb_node *rb, int color) { rb->rb_parent_color = (rb->rb_parent_color & ~1) | color; } static inline __attribute__((always_inline)) void rb_init_node(struct rb_node *rb) { rb->rb_parent_color = 0; rb->rb_right = ((void *)0); rb->rb_left = ((void *)0); (rb_set_parent(rb, rb)); } extern void rb_insert_color(struct rb_node *, struct rb_root *); extern void rb_erase(struct rb_node *, struct rb_root *); typedef void (*rb_augment_f)(struct rb_node *node, void *data); extern void rb_augment_insert(struct rb_node *node, rb_augment_f func, void *data); extern struct rb_node *rb_augment_erase_begin(struct rb_node *node); extern void rb_augment_erase_end(struct rb_node *node, rb_augment_f func, void *data); extern struct rb_node *rb_next(const struct rb_node *); extern struct rb_node *rb_prev(const struct rb_node *); extern struct rb_node *rb_first(const struct rb_root *); extern struct rb_node *rb_last(const struct rb_root *); extern void rb_replace_node(struct rb_node *victim, struct rb_node *new, struct rb_root *root); static inline __attribute__((always_inline)) void rb_link_node(struct rb_node * node, struct rb_node * parent, struct rb_node ** rb_link) { node->rb_parent_color = (unsigned long )parent; node->rb_left = node->rb_right = ((void *)0); *rb_link = node; } struct raw_prio_tree_node { struct prio_tree_node *left; struct prio_tree_node *right; struct prio_tree_node *parent; }; struct prio_tree_node { struct prio_tree_node *left; struct prio_tree_node *right; struct prio_tree_node *parent; unsigned long start; unsigned long last; }; struct prio_tree_root { struct prio_tree_node *prio_tree_node; unsigned short index_bits; unsigned short raw; }; struct prio_tree_iter { struct prio_tree_node *cur; unsigned long mask; unsigned long value; int size_level; struct prio_tree_root *root; unsigned long r_index; unsigned long h_index; }; static inline __attribute__((always_inline)) void prio_tree_iter_init(struct prio_tree_iter *iter, struct prio_tree_root *root, unsigned long r_index, unsigned long h_index) { iter->root = root; iter->r_index = r_index; iter->h_index = h_index; iter->cur = ((void *)0); } static inline __attribute__((always_inline)) int prio_tree_empty(const struct prio_tree_root *root) { return root->prio_tree_node == ((void *)0); } static inline __attribute__((always_inline)) int prio_tree_root(const struct prio_tree_node *node) { return node->parent == node; } static inline __attribute__((always_inline)) int prio_tree_left_empty(const struct prio_tree_node *node) { return node->left == node; } static inline __attribute__((always_inline)) int prio_tree_right_empty(const struct prio_tree_node *node) { return node->right == node; } struct prio_tree_node *prio_tree_replace(struct prio_tree_root *root, struct prio_tree_node *old, struct prio_tree_node *node); struct prio_tree_node *prio_tree_insert(struct prio_tree_root *root, struct prio_tree_node *node); void prio_tree_remove(struct prio_tree_root *root, struct prio_tree_node *node); struct prio_tree_node *prio_tree_next(struct prio_tree_iter *iter); enum page_debug_flags { PAGE_DEBUG_FLAG_POISON, }; struct address_space; struct page { unsigned long flags; atomic_t _count; union { atomic_t _mapcount; struct { u16 inuse; u16 objects; }; }; union { struct { unsigned long private; struct address_space *mapping; }; struct kmem_cache *slab; struct page *first_page; }; union { unsigned long index; void *freelist; }; struct list_head lru; }; typedef unsigned long vm_flags_t; struct vm_region { struct rb_node vm_rb; vm_flags_t vm_flags; unsigned long vm_start; unsigned long vm_end; unsigned long vm_top; unsigned long vm_pgoff; struct file *vm_file; int vm_usage; bool vm_icache_flushed : 1; }; struct vm_area_struct { struct mm_struct * vm_mm; unsigned long vm_start; unsigned long vm_end; struct vm_area_struct *vm_next, *vm_prev; pgprot_t vm_page_prot; unsigned long vm_flags; struct rb_node vm_rb; union { struct { struct list_head list; void *parent; struct vm_area_struct *head; } vm_set; struct raw_prio_tree_node prio_tree_node; } shared; struct list_head anon_vma_chain; struct anon_vma *anon_vma; const struct vm_operations_struct *vm_ops; unsigned long vm_pgoff; struct file * vm_file; void * vm_private_data; }; struct core_thread { struct task_struct *task; struct core_thread *next; }; struct core_state { atomic_t nr_threads; struct core_thread dumper; struct completion startup; }; enum { MM_FILEPAGES, MM_ANONPAGES, MM_SWAPENTS, NR_MM_COUNTERS }; struct mm_rss_stat { atomic_long_t count[NR_MM_COUNTERS]; }; struct mm_struct { struct vm_area_struct * mmap; struct rb_root mm_rb; struct vm_area_struct * mmap_cache; unsigned long (*get_unmapped_area) (struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags); void (*unmap_area) (struct mm_struct *mm, unsigned long addr); unsigned long mmap_base; unsigned long task_size; unsigned long cached_hole_size; unsigned long free_area_cache; pgd_t * pgd; atomic_t mm_users; atomic_t mm_count; int map_count; spinlock_t page_table_lock; struct rw_semaphore mmap_sem; struct list_head mmlist; unsigned long hiwater_rss; unsigned long hiwater_vm; unsigned long total_vm, locked_vm, shared_vm, exec_vm; unsigned long stack_vm, reserved_vm, def_flags, nr_ptes; unsigned long start_code, end_code, start_data, end_data; unsigned long start_brk, brk, start_stack; unsigned long arg_start, arg_end, env_start, env_end; unsigned long saved_auxv[(2*(2 + 19 + 1))]; struct mm_rss_stat rss_stat; struct linux_binfmt *binfmt; cpumask_var_t cpu_vm_mask_var; mm_context_t context; unsigned int faultstamp; unsigned int token_priority; unsigned int last_interval; atomic_t oom_disable_count; unsigned long flags; struct core_state *core_state; spinlock_t ioctx_lock; struct hlist_head ioctx_list; struct file *exe_file; unsigned long num_exe_file_vmas; struct mmu_notifier_mm *mmu_notifier_mm; pgtable_t pmd_huge_pte; }; static inline __attribute__((always_inline)) void mm_init_cpumask(struct mm_struct *mm) { } static inline __attribute__((always_inline)) cpumask_t *mm_cpumask(struct mm_struct *mm) { return mm->cpu_vm_mask_var; } typedef unsigned long cputime_t; typedef u64 cputime64_t; struct ipc_perm { __kernel_key_t key; __kernel_uid_t uid; __kernel_gid_t gid; __kernel_uid_t cuid; __kernel_gid_t cgid; __kernel_mode_t mode; unsigned short seq; }; struct ipc64_perm { __kernel_key_t key; __kernel_uid32_t uid; __kernel_gid32_t gid; __kernel_uid32_t cuid; __kernel_gid32_t cgid; __kernel_mode_t mode; unsigned char __pad1[4 - sizeof(__kernel_mode_t)]; unsigned short seq; unsigned short __pad2; unsigned long __unused1; unsigned long __unused2; }; struct ipc_kludge { struct msgbuf *msgp; long msgtyp; }; struct kern_ipc_perm { spinlock_t lock; int deleted; int id; key_t key; uid_t uid; gid_t gid; uid_t cuid; gid_t cgid; mode_t mode; unsigned long seq; void *security; }; struct semid_ds { struct ipc_perm sem_perm; __kernel_time_t sem_otime; __kernel_time_t sem_ctime; struct sem *sem_base; struct sem_queue *sem_pending; struct sem_queue **sem_pending_last; struct sem_undo *undo; unsigned short sem_nsems; }; struct semid64_ds { struct ipc64_perm sem_perm; __kernel_time_t sem_otime; unsigned long __unused1; __kernel_time_t sem_ctime; unsigned long __unused2; unsigned long sem_nsems; unsigned long __unused3; unsigned long __unused4; }; struct sembuf { unsigned short sem_num; short sem_op; short sem_flg; }; union semun { int val; struct semid_ds *buf; unsigned short *array; struct seminfo *__buf; void *__pad; }; struct seminfo { int semmap; int semmni; int semmns; int semmnu; int semmsl; int semopm; int semume; int semusz; int semvmx; int semaem; }; struct task_struct; struct sem { int semval; int sempid; struct list_head sem_pending; }; struct sem_array { struct kern_ipc_perm __attribute__((__aligned__((1 << (6))))) sem_perm; time_t sem_otime; time_t sem_ctime; struct sem *sem_base; struct list_head sem_pending; struct list_head list_id; int sem_nsems; int complex_count; }; struct sem_queue { struct list_head simple_list; struct list_head list; struct task_struct *sleeper; struct sem_undo *undo; int pid; int status; struct sembuf *sops; int nsops; int alter; }; struct sem_undo { struct list_head list_proc; struct rcu_head rcu; struct sem_undo_list *ulp; struct list_head list_id; int semid; short * semadj; }; struct sem_undo_list { atomic_t refcnt; spinlock_t lock; struct list_head list_proc; }; struct sysv_sem { struct sem_undo_list *undo_list; }; extern int copy_semundo(unsigned long clone_flags, struct task_struct *tsk); extern void exit_sem(struct task_struct *tsk); struct siginfo; typedef unsigned long old_sigset_t; typedef struct { unsigned long sig[(64 / 32)]; } sigset_t; typedef void __signalfn_t(int); typedef __signalfn_t *__sighandler_t; typedef void __restorefn_t(void); typedef __restorefn_t *__sigrestore_t; extern void do_notify_resume(struct pt_regs *, void *, __u32); struct old_sigaction { __sighandler_t sa_handler; old_sigset_t sa_mask; unsigned long sa_flags; __sigrestore_t sa_restorer; }; struct sigaction { __sighandler_t sa_handler; unsigned long sa_flags; __sigrestore_t sa_restorer; sigset_t sa_mask; }; struct k_sigaction { struct sigaction sa; }; typedef struct sigaltstack { void *ss_sp; int ss_flags; size_t ss_size; } stack_t; static inline __attribute__((always_inline)) void __gen_sigaddset(sigset_t *set, int _sig) { asm("btsl %1,%0" : "+m"(*set) : "Ir"(_sig - 1) : "cc"); } static inline __attribute__((always_inline)) void __const_sigaddset(sigset_t *set, int _sig) { unsigned long sig = _sig - 1; set->sig[sig / 32] |= 1 << (sig % 32); } static inline __attribute__((always_inline)) void __gen_sigdelset(sigset_t *set, int _sig) { asm("btrl %1,%0" : "+m"(*set) : "Ir"(_sig - 1) : "cc"); } static inline __attribute__((always_inline)) void __const_sigdelset(sigset_t *set, int _sig) { unsigned long sig = _sig - 1; set->sig[sig / 32] &= ~(1 << (sig % 32)); } static inline __attribute__((always_inline)) int __const_sigismember(sigset_t *set, int _sig) { unsigned long sig = _sig - 1; return 1 & (set->sig[sig / 32] >> (sig % 32)); } static inline __attribute__((always_inline)) int __gen_sigismember(sigset_t *set, int _sig) { int ret; asm("btl %2,%1\n\tsbbl %0,%0" : "=r"(ret) : "m"(*set), "Ir"(_sig-1) : "cc"); return ret; } static inline __attribute__((always_inline)) int sigfindinword(unsigned long word) { asm("bsfl %1,%0" : "=r"(word) : "rm"(word) : "cc"); return word; } struct pt_regs; typedef union sigval { int sival_int; void *sival_ptr; } sigval_t; typedef struct siginfo { int si_signo; int si_errno; int si_code; union { int _pad[((128 - (3 * sizeof(int))) / sizeof(int))]; struct { __kernel_pid_t _pid; __kernel_uid32_t _uid; } _kill; struct { __kernel_timer_t _tid; int _overrun; char _pad[sizeof( __kernel_uid32_t) - sizeof(int)]; sigval_t _sigval; int _sys_private; } _timer; struct { __kernel_pid_t _pid; __kernel_uid32_t _uid; sigval_t _sigval; } _rt; struct { __kernel_pid_t _pid; __kernel_uid32_t _uid; int _status; __kernel_clock_t _utime; __kernel_clock_t _stime; } _sigchld; struct { void *_addr; short _addr_lsb; } _sigfault; struct { long _band; int _fd; } _sigpoll; } _sifields; } siginfo_t; typedef struct sigevent { sigval_t sigev_value; int sigev_signo; int sigev_notify; union { int _pad[((64 - (sizeof(int) * 2 + sizeof(sigval_t))) / sizeof(int))]; int _tid; struct { void (*_function)(sigval_t); void *_attribute; } _sigev_thread; } _sigev_un; } sigevent_t; struct siginfo; void do_schedule_next_timer(struct siginfo *info); static inline __attribute__((always_inline)) void copy_siginfo(struct siginfo *to, struct siginfo *from) { if (__builtin_constant_p(((from->si_code < 0))) ? !!((from->si_code < 0)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/asm-generic/siginfo.h", .line = 289, }; ______r = !!((from->si_code < 0)); ______f.miss_hit[______r]++; ______r; })) __builtin_memcpy(to, from, sizeof(*to)); else __builtin_memcpy(to, from, (3 * sizeof(int)) + sizeof(from->_sifields._sigchld)); } extern int copy_siginfo_to_user(struct siginfo *to, struct siginfo *from); struct task_struct; extern int print_fatal_signals; struct sigqueue { struct list_head list; int flags; siginfo_t info; struct user_struct *user; }; struct sigpending { struct list_head list; sigset_t signal; }; static inline __attribute__((always_inline)) int sigisemptyset(sigset_t *set) { extern void _NSIG_WORDS_is_unsupported_size(void); switch ((64 / 32)) { case 4: return (set->sig[3] | set->sig[2] | set->sig[1] | set->sig[0]) == 0; case 2: return (set->sig[1] | set->sig[0]) == 0; case 1: return set->sig[0] == 0; default: _NSIG_WORDS_is_unsupported_size(); return 0; } } static inline __attribute__((always_inline)) void sigorsets(sigset_t *r, const sigset_t *a, const sigset_t *b) { extern void _NSIG_WORDS_is_unsupported_size(void); unsigned long a0, a1, a2, a3, b0, b1, b2, b3; switch ((64 / 32)) { case 4: a3 = a->sig[3]; a2 = a->sig[2]; b3 = b->sig[3]; b2 = b->sig[2]; r->sig[3] = ((a3) | (b3)); r->sig[2] = ((a2) | (b2)); case 2: a1 = a->sig[1]; b1 = b->sig[1]; r->sig[1] = ((a1) | (b1)); case 1: a0 = a->sig[0]; b0 = b->sig[0]; r->sig[0] = ((a0) | (b0)); break; default: _NSIG_WORDS_is_unsupported_size(); } } static inline __attribute__((always_inline)) void sigandsets(sigset_t *r, const sigset_t *a, const sigset_t *b) { extern void _NSIG_WORDS_is_unsupported_size(void); unsigned long a0, a1, a2, a3, b0, b1, b2, b3; switch ((64 / 32)) { case 4: a3 = a->sig[3]; a2 = a->sig[2]; b3 = b->sig[3]; b2 = b->sig[2]; r->sig[3] = ((a3) & (b3)); r->sig[2] = ((a2) & (b2)); case 2: a1 = a->sig[1]; b1 = b->sig[1]; r->sig[1] = ((a1) & (b1)); case 1: a0 = a->sig[0]; b0 = b->sig[0]; r->sig[0] = ((a0) & (b0)); break; default: _NSIG_WORDS_is_unsupported_size(); } } static inline __attribute__((always_inline)) void sigandnsets(sigset_t *r, const sigset_t *a, const sigset_t *b) { extern void _NSIG_WORDS_is_unsupported_size(void); unsigned long a0, a1, a2, a3, b0, b1, b2, b3; switch ((64 / 32)) { case 4: a3 = a->sig[3]; a2 = a->sig[2]; b3 = b->sig[3]; b2 = b->sig[2]; r->sig[3] = ((a3) & ~(b3)); r->sig[2] = ((a2) & ~(b2)); case 2: a1 = a->sig[1]; b1 = b->sig[1]; r->sig[1] = ((a1) & ~(b1)); case 1: a0 = a->sig[0]; b0 = b->sig[0]; r->sig[0] = ((a0) & ~(b0)); break; default: _NSIG_WORDS_is_unsupported_size(); } } static inline __attribute__((always_inline)) void signotset(sigset_t *set) { extern void _NSIG_WORDS_is_unsupported_size(void); switch ((64 / 32)) { case 4: set->sig[3] = (~(set->sig[3])); set->sig[2] = (~(set->sig[2])); case 2: set->sig[1] = (~(set->sig[1])); case 1: set->sig[0] = (~(set->sig[0])); break; default: _NSIG_WORDS_is_unsupported_size(); } } static inline __attribute__((always_inline)) void sigemptyset(sigset_t *set) { switch ((64 / 32)) { default: __builtin_memset(set, 0, sizeof(sigset_t)); break; case 2: set->sig[1] = 0; case 1: set->sig[0] = 0; break; } } static inline __attribute__((always_inline)) void sigfillset(sigset_t *set) { switch ((64 / 32)) { default: __builtin_memset(set, -1, sizeof(sigset_t)); break; case 2: set->sig[1] = -1; case 1: set->sig[0] = -1; break; } } static inline __attribute__((always_inline)) void sigaddsetmask(sigset_t *set, unsigned long mask) { set->sig[0] |= mask; } static inline __attribute__((always_inline)) void sigdelsetmask(sigset_t *set, unsigned long mask) { set->sig[0] &= ~mask; } static inline __attribute__((always_inline)) int sigtestsetmask(sigset_t *set, unsigned long mask) { return (set->sig[0] & mask) != 0; } static inline __attribute__((always_inline)) void siginitset(sigset_t *set, unsigned long mask) { set->sig[0] = mask; switch ((64 / 32)) { default: __builtin_memset(&set->sig[1], 0, sizeof(long)*((64 / 32)-1)); break; case 2: set->sig[1] = 0; case 1: ; } } static inline __attribute__((always_inline)) void siginitsetinv(sigset_t *set, unsigned long mask) { set->sig[0] = ~mask; switch ((64 / 32)) { default: __builtin_memset(&set->sig[1], -1, sizeof(long)*((64 / 32)-1)); break; case 2: set->sig[1] = -1; case 1: ; } } static inline __attribute__((always_inline)) void init_sigpending(struct sigpending *sig) { sigemptyset(&sig->signal); INIT_LIST_HEAD(&sig->list); } extern void flush_sigqueue(struct sigpending *queue); static inline __attribute__((always_inline)) int valid_signal(unsigned long sig) { return sig <= 64 ? 1 : 0; } struct timespec; struct pt_regs; extern int next_signal(struct sigpending *pending, sigset_t *mask); extern int do_send_sig_info(int sig, struct siginfo *info, struct task_struct *p, bool group); extern int group_send_sig_info(int sig, struct siginfo *info, struct task_struct *p); extern int __group_send_sig_info(int, struct siginfo *, struct task_struct *); extern long do_rt_tgsigqueueinfo(pid_t tgid, pid_t pid, int sig, siginfo_t *info); extern long do_sigpending(void *, unsigned long); extern int do_sigtimedwait(const sigset_t *, siginfo_t *, const struct timespec *); extern int sigprocmask(int, sigset_t *, sigset_t *); extern void set_current_blocked(const sigset_t *); extern int show_unhandled_signals; extern int get_signal_to_deliver(siginfo_t *info, struct k_sigaction *return_ka, struct pt_regs *regs, void *cookie); extern void exit_signals(struct task_struct *tsk); extern struct kmem_cache *sighand_cachep; int unhandled_signal(struct task_struct *tsk, int sig); void signals_init(void); enum pid_type { PIDTYPE_PID, PIDTYPE_PGID, PIDTYPE_SID, PIDTYPE_MAX }; struct upid { int nr; struct pid_namespace *ns; struct hlist_node pid_chain; }; struct pid { atomic_t count; unsigned int level; struct hlist_head tasks[PIDTYPE_MAX]; struct rcu_head rcu; struct upid numbers[1]; }; extern struct pid init_struct_pid; struct pid_link { struct hlist_node node; struct pid *pid; }; static inline __attribute__((always_inline)) struct pid *get_pid(struct pid *pid) { if (__builtin_constant_p(((pid))) ? !!((pid)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/pid.h", .line = 77, }; ______r = !!((pid)); ______f.miss_hit[______r]++; ______r; })) atomic_inc(&pid->count); return pid; } extern void put_pid(struct pid *pid); extern struct task_struct *pid_task(struct pid *pid, enum pid_type); extern struct task_struct *get_pid_task(struct pid *pid, enum pid_type); extern struct pid *get_task_pid(struct task_struct *task, enum pid_type type); extern void attach_pid(struct task_struct *task, enum pid_type type, struct pid *pid); extern void detach_pid(struct task_struct *task, enum pid_type); extern void change_pid(struct task_struct *task, enum pid_type, struct pid *pid); extern void transfer_pid(struct task_struct *old, struct task_struct *new, enum pid_type); struct pid_namespace; extern struct pid_namespace init_pid_ns; extern struct pid *find_pid_ns(int nr, struct pid_namespace *ns); extern struct pid *find_vpid(int nr); extern struct pid *find_get_pid(int nr); extern struct pid *find_ge_pid(int nr, struct pid_namespace *); int next_pidmap(struct pid_namespace *pid_ns, unsigned int last); extern struct pid *alloc_pid(struct pid_namespace *ns); extern void free_pid(struct pid *pid); static inline __attribute__((always_inline)) struct pid_namespace *ns_of_pid(struct pid *pid) { struct pid_namespace *ns = ((void *)0); if (__builtin_constant_p(((pid))) ? !!((pid)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/pid.h", .line = 138, }; ______r = !!((pid)); ______f.miss_hit[______r]++; ______r; })) ns = pid->numbers[pid->level].ns; return ns; } static inline __attribute__((always_inline)) bool is_child_reaper(struct pid *pid) { return pid->numbers[pid->level].nr == 1; } static inline __attribute__((always_inline)) pid_t pid_nr(struct pid *pid) { pid_t nr = 0; if (__builtin_constant_p(((pid))) ? !!((pid)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/pid.h", .line = 168, }; ______r = !!((pid)); ______f.miss_hit[______r]++; ______r; })) nr = pid->numbers[0].nr; return nr; } pid_t pid_nr_ns(struct pid *pid, struct pid_namespace *ns); pid_t pid_vnr(struct pid *pid); struct percpu_counter { spinlock_t lock; s64 count; struct list_head list; s32 *counters; }; extern int percpu_counter_batch; int __percpu_counter_init(struct percpu_counter *fbc, s64 amount, struct lock_class_key *key); void percpu_counter_destroy(struct percpu_counter *fbc); void percpu_counter_set(struct percpu_counter *fbc, s64 amount); void __percpu_counter_add(struct percpu_counter *fbc, s64 amount, s32 batch); s64 __percpu_counter_sum(struct percpu_counter *fbc); int percpu_counter_compare(struct percpu_counter *fbc, s64 rhs); static inline __attribute__((always_inline)) void percpu_counter_add(struct percpu_counter *fbc, s64 amount) { __percpu_counter_add(fbc, amount, percpu_counter_batch); } static inline __attribute__((always_inline)) s64 percpu_counter_sum_positive(struct percpu_counter *fbc) { s64 ret = __percpu_counter_sum(fbc); return ret < 0 ? 0 : ret; } static inline __attribute__((always_inline)) s64 percpu_counter_sum(struct percpu_counter *fbc) { return __percpu_counter_sum(fbc); } static inline __attribute__((always_inline)) s64 percpu_counter_read(struct percpu_counter *fbc) { return fbc->count; } static inline __attribute__((always_inline)) s64 percpu_counter_read_positive(struct percpu_counter *fbc) { s64 ret = fbc->count; __asm__ __volatile__("": : :"memory"); if (__builtin_constant_p(((ret >= 0))) ? !!((ret >= 0)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/percpu_counter.h", .line = 76, }; ______r = !!((ret >= 0)); ______f.miss_hit[______r]++; ______r; })) return ret; return 0; } static inline __attribute__((always_inline)) int percpu_counter_initialized(struct percpu_counter *fbc) { return (fbc->counters != ((void *)0)); } static inline __attribute__((always_inline)) void percpu_counter_inc(struct percpu_counter *fbc) { percpu_counter_add(fbc, 1); } static inline __attribute__((always_inline)) void percpu_counter_dec(struct percpu_counter *fbc) { percpu_counter_add(fbc, -1); } static inline __attribute__((always_inline)) void percpu_counter_sub(struct percpu_counter *fbc, s64 amount) { percpu_counter_add(fbc, -amount); } struct prop_global { int shift; struct percpu_counter events; }; struct prop_descriptor { int index; struct prop_global pg[2]; struct mutex mutex; }; int prop_descriptor_init(struct prop_descriptor *pd, int shift); void prop_change_shift(struct prop_descriptor *pd, int new_shift); struct prop_local_percpu { struct percpu_counter events; int shift; unsigned long period; spinlock_t lock; }; int prop_local_init_percpu(struct prop_local_percpu *pl); void prop_local_destroy_percpu(struct prop_local_percpu *pl); void __prop_inc_percpu(struct prop_descriptor *pd, struct prop_local_percpu *pl); void prop_fraction_percpu(struct prop_descriptor *pd, struct prop_local_percpu *pl, long *numerator, long *denominator); static inline __attribute__((always_inline)) void prop_inc_percpu(struct prop_descriptor *pd, struct prop_local_percpu *pl) { unsigned long flags; do { do { ({ unsigned long __dummy; typeof(flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); flags = arch_local_irq_save(); } while (0); trace_hardirqs_off(); } while (0); __prop_inc_percpu(pd, pl); do { if (__builtin_constant_p(((({ ({ unsigned long __dummy; typeof(flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); arch_irqs_disabled_flags(flags); })))) ? !!((({ ({ unsigned long __dummy; typeof(flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); arch_irqs_disabled_flags(flags); }))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/proportions.h", .line = 77, }; ______r = !!((({ ({ unsigned long __dummy; typeof(flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); arch_irqs_disabled_flags(flags); }))); ______f.miss_hit[______r]++; ______r; })) { do { ({ unsigned long __dummy; typeof(flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); arch_local_irq_restore(flags); } while (0); trace_hardirqs_off(); } else { trace_hardirqs_on(); do { ({ unsigned long __dummy; typeof(flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); arch_local_irq_restore(flags); } while (0); } } while (0); } void __prop_inc_percpu_max(struct prop_descriptor *pd, struct prop_local_percpu *pl, long frac); struct prop_local_single { unsigned long events; unsigned long period; int shift; spinlock_t lock; }; int prop_local_init_single(struct prop_local_single *pl); void prop_local_destroy_single(struct prop_local_single *pl); void __prop_inc_single(struct prop_descriptor *pd, struct prop_local_single *pl); void prop_fraction_single(struct prop_descriptor *pd, struct prop_local_single *pl, long *numerator, long *denominator); static inline __attribute__((always_inline)) void prop_inc_single(struct prop_descriptor *pd, struct prop_local_single *pl) { unsigned long flags; do { do { ({ unsigned long __dummy; typeof(flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); flags = arch_local_irq_save(); } while (0); trace_hardirqs_off(); } while (0); __prop_inc_single(pd, pl); do { if (__builtin_constant_p(((({ ({ unsigned long __dummy; typeof(flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); arch_irqs_disabled_flags(flags); })))) ? !!((({ ({ unsigned long __dummy; typeof(flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); arch_irqs_disabled_flags(flags); }))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/proportions.h", .line = 129, }; ______r = !!((({ ({ unsigned long __dummy; typeof(flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); arch_irqs_disabled_flags(flags); }))); ______f.miss_hit[______r]++; ______r; })) { do { ({ unsigned long __dummy; typeof(flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); arch_local_irq_restore(flags); } while (0); trace_hardirqs_off(); } else { trace_hardirqs_on(); do { ({ unsigned long __dummy; typeof(flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); arch_local_irq_restore(flags); } while (0); } } while (0); } typedef struct { int mode; } seccomp_t; extern void __secure_computing(int); static inline __attribute__((always_inline)) void secure_computing(int this_syscall) { if (__builtin_constant_p((((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 8)) ? !!(test_ti_thread_flag(current_thread_info(), 8)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/seccomp.h", .line = 15, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 8)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 8)) ? !!(test_ti_thread_flag(current_thread_info(), 8)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/seccomp.h", .line = 15, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 8)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/seccomp.h", .line = 15, }; ______r = !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 8)) ? !!(test_ti_thread_flag(current_thread_info(), 8)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/seccomp.h", .line = 15, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 8)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) __secure_computing(this_syscall); } extern long prctl_get_seccomp(void); extern long prctl_set_seccomp(unsigned long); static inline __attribute__((always_inline)) void __list_add_rcu(struct list_head *new, struct list_head *prev, struct list_head *next) { new->next = next; new->prev = prev; ({ if (__builtin_constant_p(((!__builtin_constant_p((new)) || (((new)) != ((void *)0))))) ? !!((!__builtin_constant_p((new)) || (((new)) != ((void *)0)))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/rculist.h", .line = 38, }; ______r = !!((!__builtin_constant_p((new)) || (((new)) != ((void *)0)))); ______f.miss_hit[______r]++; ______r; })) __asm__ __volatile__("": : :"memory"); (((*((struct list_head **)(&(prev)->next))))) = (typeof(*(new)) *)((new)); }); next->prev = new; } static inline __attribute__((always_inline)) void list_add_rcu(struct list_head *new, struct list_head *head) { __list_add_rcu(new, head, head->next); } static inline __attribute__((always_inline)) void list_add_tail_rcu(struct list_head *new, struct list_head *head) { __list_add_rcu(new, head->prev, head); } static inline __attribute__((always_inline)) void list_del_rcu(struct list_head *entry) { __list_del(entry->prev, entry->next); entry->prev = ((void *) 0x00200200 + (0x0UL)); } static inline __attribute__((always_inline)) void hlist_del_init_rcu(struct hlist_node *n) { if (__builtin_constant_p(((!hlist_unhashed(n)))) ? !!((!hlist_unhashed(n))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/rculist.h", .line = 137, }; ______r = !!((!hlist_unhashed(n))); ______f.miss_hit[______r]++; ______r; })) { __hlist_del(n); n->pprev = ((void *)0); } } static inline __attribute__((always_inline)) void list_replace_rcu(struct list_head *old, struct list_head *new) { new->next = old->next; new->prev = old->prev; ({ if (__builtin_constant_p(((!__builtin_constant_p((new)) || (((new)) != ((void *)0))))) ? !!((!__builtin_constant_p((new)) || (((new)) != ((void *)0)))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/rculist.h", .line = 156, }; ______r = !!((!__builtin_constant_p((new)) || (((new)) != ((void *)0)))); ______f.miss_hit[______r]++; ______r; })) __asm__ __volatile__("": : :"memory"); (((*((struct list_head **)(&(new->prev)->next))))) = (typeof(*(new)) *)((new)); }); new->next->prev = new; old->prev = ((void *) 0x00200200 + (0x0UL)); } static inline __attribute__((always_inline)) void list_splice_init_rcu(struct list_head *list, struct list_head *head, void (*sync)(void)) { struct list_head *first = list->next; struct list_head *last = list->prev; struct list_head *at = head->next; if (__builtin_constant_p(((list_empty(head)))) ? !!((list_empty(head))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/rculist.h", .line = 186, }; ______r = !!((list_empty(head))); ______f.miss_hit[______r]++; ______r; })) return; INIT_LIST_HEAD(list); sync(); last->next = at; ({ if (__builtin_constant_p(((!__builtin_constant_p((first)) || (((first)) != ((void *)0))))) ? !!((!__builtin_constant_p((first)) || (((first)) != ((void *)0)))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/rculist.h", .line = 211, }; ______r = !!((!__builtin_constant_p((first)) || (((first)) != ((void *)0)))); ______f.miss_hit[______r]++; ______r; })) __asm__ __volatile__("": : :"memory"); (((*((struct list_head **)(&(head)->next))))) = (typeof(*(first)) *)((first)); }); first->prev = head; at->prev = last; } static inline __attribute__((always_inline)) void hlist_del_rcu(struct hlist_node *n) { __hlist_del(n); n->pprev = ((void *) 0x00200200 + (0x0UL)); } static inline __attribute__((always_inline)) void hlist_replace_rcu(struct hlist_node *old, struct hlist_node *new) { struct hlist_node *next = old->next; new->next = next; new->pprev = old->pprev; ({ if (__builtin_constant_p(((!__builtin_constant_p((new)) || (((new)) != ((void *)0))))) ? !!((!__builtin_constant_p((new)) || (((new)) != ((void *)0)))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/rculist.h", .line = 329, }; ______r = !!((!__builtin_constant_p((new)) || (((new)) != ((void *)0)))); ______f.miss_hit[______r]++; ______r; })) __asm__ __volatile__("": : :"memory"); ((*(struct hlist_node **)new->pprev)) = (typeof(*(new)) *)((new)); }); if (__builtin_constant_p(((next))) ? !!((next)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/rculist.h", .line = 330, }; ______r = !!((next)); ______f.miss_hit[______r]++; ______r; })) new->next->pprev = &new->next; old->pprev = ((void *) 0x00200200 + (0x0UL)); } static inline __attribute__((always_inline)) void hlist_add_head_rcu(struct hlist_node *n, struct hlist_head *h) { struct hlist_node *first = h->first; n->next = first; n->pprev = &h->first; ({ if (__builtin_constant_p(((!__builtin_constant_p((n)) || (((n)) != ((void *)0))))) ? !!((!__builtin_constant_p((n)) || (((n)) != ((void *)0)))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/rculist.h", .line = 368, }; ______r = !!((!__builtin_constant_p((n)) || (((n)) != ((void *)0)))); ______f.miss_hit[______r]++; ______r; })) __asm__ __volatile__("": : :"memory"); (((*((struct hlist_node **)(&(h)->first))))) = (typeof(*(n)) *)((n)); }); if (__builtin_constant_p(((first))) ? !!((first)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/rculist.h", .line = 369, }; ______r = !!((first)); ______f.miss_hit[______r]++; ______r; })) first->pprev = &n->next; } static inline __attribute__((always_inline)) void hlist_add_before_rcu(struct hlist_node *n, struct hlist_node *next) { n->pprev = next->pprev; n->next = next; ({ if (__builtin_constant_p(((!__builtin_constant_p((n)) || (((n)) != ((void *)0))))) ? !!((!__builtin_constant_p((n)) || (((n)) != ((void *)0)))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/rculist.h", .line = 396, }; ______r = !!((!__builtin_constant_p((n)) || (((n)) != ((void *)0)))); ______f.miss_hit[______r]++; ______r; })) __asm__ __volatile__("": : :"memory"); (((*((struct hlist_node **)((n)->pprev))))) = (typeof(*(n)) *)((n)); }); next->pprev = &n->next; } static inline __attribute__((always_inline)) void hlist_add_after_rcu(struct hlist_node *prev, struct hlist_node *n) { n->next = prev->next; n->pprev = &prev->next; ({ if (__builtin_constant_p(((!__builtin_constant_p((n)) || (((n)) != ((void *)0))))) ? !!((!__builtin_constant_p((n)) || (((n)) != ((void *)0)))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/rculist.h", .line = 423, }; ______r = !!((!__builtin_constant_p((n)) || (((n)) != ((void *)0)))); ______f.miss_hit[______r]++; ______r; })) __asm__ __volatile__("": : :"memory"); (((*((struct hlist_node **)(&(prev)->next))))) = (typeof(*(n)) *)((n)); }); if (__builtin_constant_p(((n->next))) ? !!((n->next)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/rculist.h", .line = 424, }; ______r = !!((n->next)); ______f.miss_hit[______r]++; ______r; })) n->next->pprev = &n->next; } struct plist_head { struct list_head node_list; raw_spinlock_t *rawlock; spinlock_t *spinlock; }; struct plist_node { int prio; struct list_head prio_list; struct list_head node_list; }; static inline __attribute__((always_inline)) void plist_head_init(struct plist_head *head, spinlock_t *lock) { INIT_LIST_HEAD(&head->node_list); head->spinlock = lock; head->rawlock = ((void *)0); } static inline __attribute__((always_inline)) void plist_head_init_raw(struct plist_head *head, raw_spinlock_t *lock) { INIT_LIST_HEAD(&head->node_list); head->rawlock = lock; head->spinlock = ((void *)0); } static inline __attribute__((always_inline)) void plist_node_init(struct plist_node *node, int prio) { node->prio = prio; INIT_LIST_HEAD(&node->prio_list); INIT_LIST_HEAD(&node->node_list); } extern void plist_add(struct plist_node *node, struct plist_head *head); extern void plist_del(struct plist_node *node, struct plist_head *head); static inline __attribute__((always_inline)) int plist_head_empty(const struct plist_head *head) { return list_empty(&head->node_list); } static inline __attribute__((always_inline)) int plist_node_empty(const struct plist_node *node) { return list_empty(&node->node_list); } static inline __attribute__((always_inline)) struct plist_node *plist_first(const struct plist_head *head) { return ({ const typeof( ((struct plist_node *)0)->node_list ) *__mptr = (head->node_list.next); (struct plist_node *)( (char *)__mptr - __builtin_offsetof(struct plist_node,node_list) );}) ; } static inline __attribute__((always_inline)) struct plist_node *plist_last(const struct plist_head *head) { return ({ const typeof( ((struct plist_node *)0)->node_list ) *__mptr = (head->node_list.prev); (struct plist_node *)( (char *)__mptr - __builtin_offsetof(struct plist_node,node_list) );}) ; } extern int max_lock_depth; struct rt_mutex { raw_spinlock_t wait_lock; struct plist_head wait_list; struct task_struct *owner; int save_state; const char *name, *file; int line; void *magic; }; struct rt_mutex_waiter; struct hrtimer_sleeper; extern int rt_mutex_debug_check_no_locks_freed(const void *from, unsigned long len); extern void rt_mutex_debug_check_no_locks_held(struct task_struct *task); extern void rt_mutex_debug_task_free(struct task_struct *tsk); static inline __attribute__((always_inline)) int rt_mutex_is_locked(struct rt_mutex *lock) { return lock->owner != ((void *)0); } extern void __rt_mutex_init(struct rt_mutex *lock, const char *name); extern void rt_mutex_destroy(struct rt_mutex *lock); extern void rt_mutex_lock(struct rt_mutex *lock); extern int rt_mutex_lock_interruptible(struct rt_mutex *lock, int detect_deadlock); extern int rt_mutex_timed_lock(struct rt_mutex *lock, struct hrtimer_sleeper *timeout, int detect_deadlock); extern int rt_mutex_trylock(struct rt_mutex *lock); extern void rt_mutex_unlock(struct rt_mutex *lock); struct rusage { struct timeval ru_utime; struct timeval ru_stime; long ru_maxrss; long ru_ixrss; long ru_idrss; long ru_isrss; long ru_minflt; long ru_majflt; long ru_nswap; long ru_inblock; long ru_oublock; long ru_msgsnd; long ru_msgrcv; long ru_nsignals; long ru_nvcsw; long ru_nivcsw; }; struct rlimit { unsigned long rlim_cur; unsigned long rlim_max; }; struct rlimit64 { __u64 rlim_cur; __u64 rlim_max; }; struct task_struct; int getrusage(struct task_struct *p, int who, struct rusage *ru); int do_prlimit(struct task_struct *tsk, unsigned int resource, struct rlimit *new_rlim, struct rlimit *old_rlim); struct timerqueue_node { struct rb_node node; ktime_t expires; }; struct timerqueue_head { struct rb_root head; struct timerqueue_node *next; }; extern void timerqueue_add(struct timerqueue_head *head, struct timerqueue_node *node); extern void timerqueue_del(struct timerqueue_head *head, struct timerqueue_node *node); extern struct timerqueue_node *timerqueue_iterate_next( struct timerqueue_node *node); static inline __attribute__((always_inline)) struct timerqueue_node *timerqueue_getnext(struct timerqueue_head *head) { return head->next; } static inline __attribute__((always_inline)) void timerqueue_init(struct timerqueue_node *node) { rb_init_node(&node->node); } static inline __attribute__((always_inline)) void timerqueue_init_head(struct timerqueue_head *head) { head->head = (struct rb_root) { ((void *)0), }; head->next = ((void *)0); } struct hrtimer_clock_base; struct hrtimer_cpu_base; enum hrtimer_mode { HRTIMER_MODE_ABS = 0x0, HRTIMER_MODE_REL = 0x1, HRTIMER_MODE_PINNED = 0x02, HRTIMER_MODE_ABS_PINNED = 0x02, HRTIMER_MODE_REL_PINNED = 0x03, }; enum hrtimer_restart { HRTIMER_NORESTART, HRTIMER_RESTART, }; struct hrtimer { struct timerqueue_node node; ktime_t _softexpires; enum hrtimer_restart (*function)(struct hrtimer *); struct hrtimer_clock_base *base; unsigned long state; int start_pid; void *start_site; char start_comm[16]; }; struct hrtimer_sleeper { struct hrtimer timer; struct task_struct *task; }; struct hrtimer_clock_base { struct hrtimer_cpu_base *cpu_base; int index; clockid_t clockid; struct timerqueue_head active; ktime_t resolution; ktime_t (*get_time)(void); ktime_t softirq_time; ktime_t offset; }; enum hrtimer_base_type { HRTIMER_BASE_MONOTONIC, HRTIMER_BASE_REALTIME, HRTIMER_BASE_BOOTTIME, HRTIMER_MAX_CLOCK_BASES, }; struct hrtimer_cpu_base { raw_spinlock_t lock; unsigned long active_bases; ktime_t expires_next; int hres_active; int hang_detected; unsigned long nr_events; unsigned long nr_retries; unsigned long nr_hangs; ktime_t max_hang_time; struct hrtimer_clock_base clock_base[HRTIMER_MAX_CLOCK_BASES]; }; static inline __attribute__((always_inline)) void hrtimer_set_expires(struct hrtimer *timer, ktime_t time) { timer->node.expires = time; timer->_softexpires = time; } static inline __attribute__((always_inline)) void hrtimer_set_expires_range(struct hrtimer *timer, ktime_t time, ktime_t delta) { timer->_softexpires = time; timer->node.expires = ktime_add_safe(time, delta); } static inline __attribute__((always_inline)) void hrtimer_set_expires_range_ns(struct hrtimer *timer, ktime_t time, unsigned long delta) { timer->_softexpires = time; timer->node.expires = ktime_add_safe(time, ns_to_ktime(delta)); } static inline __attribute__((always_inline)) void hrtimer_set_expires_tv64(struct hrtimer *timer, s64 tv64) { timer->node.expires.tv64 = tv64; timer->_softexpires.tv64 = tv64; } static inline __attribute__((always_inline)) void hrtimer_add_expires(struct hrtimer *timer, ktime_t time) { timer->node.expires = ktime_add_safe(timer->node.expires, time); timer->_softexpires = ktime_add_safe(timer->_softexpires, time); } static inline __attribute__((always_inline)) void hrtimer_add_expires_ns(struct hrtimer *timer, u64 ns) { timer->node.expires = ({ (ktime_t){ .tv64 = (timer->node.expires).tv64 + (ns) }; }); timer->_softexpires = ({ (ktime_t){ .tv64 = (timer->_softexpires).tv64 + (ns) }; }); } static inline __attribute__((always_inline)) ktime_t hrtimer_get_expires(const struct hrtimer *timer) { return timer->node.expires; } static inline __attribute__((always_inline)) ktime_t hrtimer_get_softexpires(const struct hrtimer *timer) { return timer->_softexpires; } static inline __attribute__((always_inline)) s64 hrtimer_get_expires_tv64(const struct hrtimer *timer) { return timer->node.expires.tv64; } static inline __attribute__((always_inline)) s64 hrtimer_get_softexpires_tv64(const struct hrtimer *timer) { return timer->_softexpires.tv64; } static inline __attribute__((always_inline)) s64 hrtimer_get_expires_ns(const struct hrtimer *timer) { return ((timer->node.expires).tv64); } static inline __attribute__((always_inline)) ktime_t hrtimer_expires_remaining(const struct hrtimer *timer) { return ({ (ktime_t){ .tv64 = (timer->node.expires).tv64 - (timer->base->get_time()).tv64 }; }); } struct clock_event_device; extern void hrtimer_interrupt(struct clock_event_device *dev); static inline __attribute__((always_inline)) ktime_t hrtimer_cb_get_time(struct hrtimer *timer) { return timer->base->get_time(); } static inline __attribute__((always_inline)) int hrtimer_is_hres_active(struct hrtimer *timer) { return timer->base->cpu_base->hres_active; } extern void hrtimer_peek_ahead_timers(void); extern void clock_was_set(void); extern void timerfd_clock_was_set(void); extern void hrtimers_resume(void); extern ktime_t ktime_get(void); extern ktime_t ktime_get_real(void); extern ktime_t ktime_get_boottime(void); extern ktime_t ktime_get_monotonic_offset(void); extern __attribute__((section(".data..percpu" ""))) __typeof__(struct tick_device) tick_cpu_device; extern void hrtimer_init(struct hrtimer *timer, clockid_t which_clock, enum hrtimer_mode mode); static inline __attribute__((always_inline)) void hrtimer_init_on_stack(struct hrtimer *timer, clockid_t which_clock, enum hrtimer_mode mode) { hrtimer_init(timer, which_clock, mode); } static inline __attribute__((always_inline)) void destroy_hrtimer_on_stack(struct hrtimer *timer) { } extern int hrtimer_start(struct hrtimer *timer, ktime_t tim, const enum hrtimer_mode mode); extern int hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, unsigned long range_ns, const enum hrtimer_mode mode); extern int __hrtimer_start_range_ns(struct hrtimer *timer, ktime_t tim, unsigned long delta_ns, const enum hrtimer_mode mode, int wakeup); extern int hrtimer_cancel(struct hrtimer *timer); extern int hrtimer_try_to_cancel(struct hrtimer *timer); static inline __attribute__((always_inline)) int hrtimer_start_expires(struct hrtimer *timer, enum hrtimer_mode mode) { unsigned long delta; ktime_t soft, hard; soft = hrtimer_get_softexpires(timer); hard = hrtimer_get_expires(timer); delta = ((({ (ktime_t){ .tv64 = (hard).tv64 - (soft).tv64 }; })).tv64); return hrtimer_start_range_ns(timer, soft, delta, mode); } static inline __attribute__((always_inline)) int hrtimer_restart(struct hrtimer *timer) { return hrtimer_start_expires(timer, HRTIMER_MODE_ABS); } extern ktime_t hrtimer_get_remaining(const struct hrtimer *timer); extern int hrtimer_get_res(const clockid_t which_clock, struct timespec *tp); extern ktime_t hrtimer_get_next_event(void); static inline __attribute__((always_inline)) int hrtimer_active(const struct hrtimer *timer) { return timer->state != 0x00; } static inline __attribute__((always_inline)) int hrtimer_is_queued(struct hrtimer *timer) { return timer->state & 0x01; } static inline __attribute__((always_inline)) int hrtimer_callback_running(struct hrtimer *timer) { return timer->state & 0x02; } extern u64 hrtimer_forward(struct hrtimer *timer, ktime_t now, ktime_t interval); static inline __attribute__((always_inline)) u64 hrtimer_forward_now(struct hrtimer *timer, ktime_t interval) { return hrtimer_forward(timer, timer->base->get_time(), interval); } extern long hrtimer_nanosleep(struct timespec *rqtp, struct timespec *rmtp, const enum hrtimer_mode mode, const clockid_t clockid); extern long hrtimer_nanosleep_restart(struct restart_block *restart_block); extern void hrtimer_init_sleeper(struct hrtimer_sleeper *sl, struct task_struct *tsk); extern int schedule_hrtimeout_range(ktime_t *expires, unsigned long delta, const enum hrtimer_mode mode); extern int schedule_hrtimeout_range_clock(ktime_t *expires, unsigned long delta, const enum hrtimer_mode mode, int clock); extern int schedule_hrtimeout(ktime_t *expires, const enum hrtimer_mode mode); extern void hrtimer_run_queues(void); extern void hrtimer_run_pending(void); extern void __attribute__ ((__section__(".init.text"))) __attribute__((__cold__)) __attribute__((no_instrument_function)) hrtimers_init(void); extern u64 ktime_divns(const ktime_t kt, s64 div); extern void sysrq_timer_list_show(void); struct task_io_accounting { u64 rchar; u64 wchar; u64 syscr; u64 syscw; u64 read_bytes; u64 write_bytes; u64 cancelled_write_bytes; }; struct latency_record { unsigned long backtrace[12]; unsigned int count; unsigned long time; unsigned long max; }; struct task_struct; extern int latencytop_enabled; void __account_scheduler_latency(struct task_struct *task, int usecs, int inter); static inline __attribute__((always_inline)) void account_scheduler_latency(struct task_struct *task, int usecs, int inter) { if (__builtin_constant_p((((__builtin_constant_p(latencytop_enabled) ? !!(latencytop_enabled) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/latencytop.h", .line = 33, }; ______r = __builtin_expect(!!(latencytop_enabled), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(latencytop_enabled) ? !!(latencytop_enabled) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/latencytop.h", .line = 33, }; ______r = __builtin_expect(!!(latencytop_enabled), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/latencytop.h", .line = 33, }; ______r = !!(((__builtin_constant_p(latencytop_enabled) ? !!(latencytop_enabled) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/latencytop.h", .line = 33, }; ______r = __builtin_expect(!!(latencytop_enabled), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) __account_scheduler_latency(task, usecs, inter); } void clear_all_latency_tracing(struct task_struct *p); typedef int32_t key_serial_t; typedef uint32_t key_perm_t; struct key; struct selinux_audit_rule; struct audit_context; struct kern_ipc_perm; static inline __attribute__((always_inline)) bool selinux_is_enabled(void) { return false; } struct user_struct; struct cred; struct inode; struct group_info { atomic_t usage; int ngroups; int nblocks; gid_t small_block[32]; gid_t *blocks[0]; }; static inline __attribute__((always_inline)) struct group_info *get_group_info(struct group_info *gi) { atomic_inc(&gi->usage); return gi; } extern struct group_info *groups_alloc(int); extern struct group_info init_groups; extern void groups_free(struct group_info *); extern int set_current_groups(struct group_info *); extern int set_groups(struct cred *, struct group_info *); extern int groups_search(const struct group_info *, gid_t); extern int in_group_p(gid_t); extern int in_egroup_p(gid_t); struct cred { atomic_t usage; uid_t uid; gid_t gid; uid_t suid; gid_t sgid; uid_t euid; gid_t egid; uid_t fsuid; gid_t fsgid; unsigned securebits; kernel_cap_t cap_inheritable; kernel_cap_t cap_permitted; kernel_cap_t cap_effective; kernel_cap_t cap_bset; void *security; struct user_struct *user; struct user_namespace *user_ns; struct group_info *group_info; struct rcu_head rcu; }; extern void __put_cred(struct cred *); extern void exit_creds(struct task_struct *); extern int copy_creds(struct task_struct *, unsigned long); extern const struct cred *get_task_cred(struct task_struct *); extern struct cred *cred_alloc_blank(void); extern struct cred *prepare_creds(void); extern struct cred *prepare_exec_creds(void); extern int commit_creds(struct cred *); extern void abort_creds(struct cred *); extern const struct cred *override_creds(const struct cred *); extern void revert_creds(const struct cred *); extern struct cred *prepare_kernel_cred(struct task_struct *); extern int change_create_files_as(struct cred *, struct inode *); extern int set_security_override(struct cred *, u32); extern int set_security_override_from_ctx(struct cred *, const char *); extern int set_create_files_as(struct cred *, struct inode *); extern void __attribute__ ((__section__(".init.text"))) __attribute__((__cold__)) __attribute__((no_instrument_function)) cred_init(void); static inline __attribute__((always_inline)) void validate_creds(const struct cred *cred) { } static inline __attribute__((always_inline)) void validate_creds_for_do_exit(struct task_struct *tsk) { } static inline __attribute__((always_inline)) void validate_process_creds(void) { } static inline __attribute__((always_inline)) struct cred *get_new_cred(struct cred *cred) { atomic_inc(&cred->usage); return cred; } static inline __attribute__((always_inline)) const struct cred *get_cred(const struct cred *cred) { struct cred *nonconst_cred = (struct cred *) cred; validate_creds(cred); return get_new_cred(nonconst_cred); } static inline __attribute__((always_inline)) void put_cred(const struct cred *_cred) { struct cred *cred = (struct cred *) _cred; validate_creds(cred); if (__builtin_constant_p(((atomic_dec_and_test(&(cred)->usage)))) ? !!((atomic_dec_and_test(&(cred)->usage))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/cred.h", .line = 261, }; ______r = !!((atomic_dec_and_test(&(cred)->usage))); ______f.miss_hit[______r]++; ______r; })) __put_cred(cred); } extern struct user_namespace init_user_ns; struct exec_domain; struct futex_pi_state; struct robust_list_head; struct bio_list; struct fs_struct; struct perf_event_context; struct blk_plug; extern unsigned long avenrun[]; extern void get_avenrun(unsigned long *loads, unsigned long offset, int shift); extern unsigned long total_forks; extern int nr_threads; extern __attribute__((section(".data..percpu" ""))) __typeof__(unsigned long) process_counts; extern int nr_processes(void); extern unsigned long nr_running(void); extern unsigned long nr_uninterruptible(void); extern unsigned long nr_iowait(void); extern unsigned long nr_iowait_cpu(int cpu); extern unsigned long this_cpu_load(void); extern void calc_global_load(unsigned long ticks); extern unsigned long get_parent_ip(unsigned long addr); struct seq_file; struct cfs_rq; struct task_group; extern void proc_sched_show_task(struct task_struct *p, struct seq_file *m); extern void proc_sched_set_task(struct task_struct *p); extern void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq); extern char ___assert_task_state[1 - 2*!!( sizeof("RSDTtZXxKW")-1 != ( __builtin_constant_p(512) ? ( (512) < 1 ? ____ilog2_NaN() : (512) & (1ULL << 63) ? 63 : (512) & (1ULL << 62) ? 62 : (512) & (1ULL << 61) ? 61 : (512) & (1ULL << 60) ? 60 : (512) & (1ULL << 59) ? 59 : (512) & (1ULL << 58) ? 58 : (512) & (1ULL << 57) ? 57 : (512) & (1ULL << 56) ? 56 : (512) & (1ULL << 55) ? 55 : (512) & (1ULL << 54) ? 54 : (512) & (1ULL << 53) ? 53 : (512) & (1ULL << 52) ? 52 : (512) & (1ULL << 51) ? 51 : (512) & (1ULL << 50) ? 50 : (512) & (1ULL << 49) ? 49 : (512) & (1ULL << 48) ? 48 : (512) & (1ULL << 47) ? 47 : (512) & (1ULL << 46) ? 46 : (512) & (1ULL << 45) ? 45 : (512) & (1ULL << 44) ? 44 : (512) & (1ULL << 43) ? 43 : (512) & (1ULL << 42) ? 42 : (512) & (1ULL << 41) ? 41 : (512) & (1ULL << 40) ? 40 : (512) & (1ULL << 39) ? 39 : (512) & (1ULL << 38) ? 38 : (512) & (1ULL << 37) ? 37 : (512) & (1ULL << 36) ? 36 : (512) & (1ULL << 35) ? 35 : (512) & (1ULL << 34) ? 34 : (512) & (1ULL << 33) ? 33 : (512) & (1ULL << 32) ? 32 : (512) & (1ULL << 31) ? 31 : (512) & (1ULL << 30) ? 30 : (512) & (1ULL << 29) ? 29 : (512) & (1ULL << 28) ? 28 : (512) & (1ULL << 27) ? 27 : (512) & (1ULL << 26) ? 26 : (512) & (1ULL << 25) ? 25 : (512) & (1ULL << 24) ? 24 : (512) & (1ULL << 23) ? 23 : (512) & (1ULL << 22) ? 22 : (512) & (1ULL << 21) ? 21 : (512) & (1ULL << 20) ? 20 : (512) & (1ULL << 19) ? 19 : (512) & (1ULL << 18) ? 18 : (512) & (1ULL << 17) ? 17 : (512) & (1ULL << 16) ? 16 : (512) & (1ULL << 15) ? 15 : (512) & (1ULL << 14) ? 14 : (512) & (1ULL << 13) ? 13 : (512) & (1ULL << 12) ? 12 : (512) & (1ULL << 11) ? 11 : (512) & (1ULL << 10) ? 10 : (512) & (1ULL << 9) ? 9 : (512) & (1ULL << 8) ? 8 : (512) & (1ULL << 7) ? 7 : (512) & (1ULL << 6) ? 6 : (512) & (1ULL << 5) ? 5 : (512) & (1ULL << 4) ? 4 : (512) & (1ULL << 3) ? 3 : (512) & (1ULL << 2) ? 2 : (512) & (1ULL << 1) ? 1 : (512) & (1ULL << 0) ? 0 : ____ilog2_NaN() ) : (sizeof(512) <= 4) ? __ilog2_u32(512) : __ilog2_u64(512) )+1)]; extern rwlock_t tasklist_lock; extern spinlock_t mmlist_lock; struct task_struct; extern void sched_init(void); extern void sched_init_smp(void); extern __attribute__((regparm(0))) void schedule_tail(struct task_struct *prev); extern void init_idle(struct task_struct *idle, int cpu); extern void init_idle_bootup_task(struct task_struct *idle); extern int runqueue_is_locked(int cpu); extern cpumask_var_t nohz_cpu_mask; static inline __attribute__((always_inline)) void select_nohz_load_balancer(int stop_tick) { } extern void show_state_filter(unsigned long state_filter); static inline __attribute__((always_inline)) void show_state(void) { show_state_filter(0); } extern void show_regs(struct pt_regs *); extern void show_stack(struct task_struct *task, unsigned long *sp); void io_schedule(void); long io_schedule_timeout(long timeout); extern void cpu_init (void); extern void trap_init(void); extern void update_process_times(int user); extern void scheduler_tick(void); extern void sched_show_task(struct task_struct *p); extern void touch_softlockup_watchdog(void); extern void touch_softlockup_watchdog_sync(void); extern void touch_all_softlockup_watchdogs(void); extern int proc_dowatchdog_thresh(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos); extern unsigned int softlockup_panic; void lockup_detector_init(void); extern unsigned int sysctl_hung_task_panic; extern unsigned long sysctl_hung_task_check_count; extern unsigned long sysctl_hung_task_timeout_secs; extern unsigned long sysctl_hung_task_warnings; extern int proc_dohung_task_timeout_secs(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos); extern char __sched_text_start[], __sched_text_end[]; extern int in_sched_functions(unsigned long addr); extern signed long schedule_timeout(signed long timeout); extern signed long schedule_timeout_interruptible(signed long timeout); extern signed long schedule_timeout_killable(signed long timeout); extern signed long schedule_timeout_uninterruptible(signed long timeout); __attribute__((regparm(0))) void schedule(void); extern int mutex_spin_on_owner(struct mutex *lock, struct task_struct *owner); struct nsproxy; struct user_namespace; extern int sysctl_max_map_count; typedef unsigned long aio_context_t; enum { IOCB_CMD_PREAD = 0, IOCB_CMD_PWRITE = 1, IOCB_CMD_FSYNC = 2, IOCB_CMD_FDSYNC = 3, IOCB_CMD_NOOP = 6, IOCB_CMD_PREADV = 7, IOCB_CMD_PWRITEV = 8, }; struct io_event { __u64 data; __u64 obj; __s64 res; __s64 res2; }; struct iocb { __u64 aio_data; __u32 aio_key, aio_reserved1; __u16 aio_lio_opcode; __s16 aio_reqprio; __u32 aio_fildes; __u64 aio_buf; __u64 aio_nbytes; __s64 aio_offset; __u64 aio_reserved2; __u32 aio_flags; __u32 aio_resfd; }; struct iovec { void *iov_base; __kernel_size_t iov_len; }; struct kvec { void *iov_base; size_t iov_len; }; static inline __attribute__((always_inline)) size_t iov_length(const struct iovec *iov, unsigned long nr_segs) { unsigned long seg; size_t ret = 0; for (seg = 0; seg < nr_segs; seg++) ret += iov[seg].iov_len; return ret; } unsigned long iov_shorten(struct iovec *iov, unsigned long nr_segs, size_t to); struct kioctx; struct kiocb { struct list_head ki_run_list; unsigned long ki_flags; int ki_users; unsigned ki_key; struct file *ki_filp; struct kioctx *ki_ctx; int (*ki_cancel)(struct kiocb *, struct io_event *); ssize_t (*ki_retry)(struct kiocb *); void (*ki_dtor)(struct kiocb *); union { void *user; struct task_struct *tsk; } ki_obj; __u64 ki_user_data; loff_t ki_pos; void *private; unsigned short ki_opcode; size_t ki_nbytes; char *ki_buf; size_t ki_left; struct iovec ki_inline_vec; struct iovec *ki_iovec; unsigned long ki_nr_segs; unsigned long ki_cur_seg; struct list_head ki_list; struct eventfd_ctx *ki_eventfd; }; struct aio_ring { unsigned id; unsigned nr; unsigned head; unsigned tail; unsigned magic; unsigned compat_features; unsigned incompat_features; unsigned header_length; struct io_event io_events[0]; }; struct aio_ring_info { unsigned long mmap_base; unsigned long mmap_size; struct page **ring_pages; spinlock_t ring_lock; long nr_pages; unsigned nr, tail; struct page *internal_pages[8]; }; struct kioctx { atomic_t users; int dead; struct mm_struct *mm; unsigned long user_id; struct hlist_node list; wait_queue_head_t wait; spinlock_t ctx_lock; int reqs_active; struct list_head active_reqs; struct list_head run_list; unsigned max_reqs; struct aio_ring_info ring_info; struct delayed_work wq; struct rcu_head rcu_head; }; extern unsigned aio_max_size; extern ssize_t wait_on_sync_kiocb(struct kiocb *iocb); extern int aio_put_req(struct kiocb *iocb); extern void kick_iocb(struct kiocb *iocb); extern int aio_complete(struct kiocb *iocb, long res, long res2); struct mm_struct; extern void exit_aio(struct mm_struct *mm); extern long do_io_submit(aio_context_t ctx_id, long nr, struct iocb * *iocbpp, bool compat); static inline __attribute__((always_inline)) struct kiocb *list_kiocb(struct list_head *h) { return ({ const typeof( ((struct kiocb *)0)->ki_list ) *__mptr = (h); (struct kiocb *)( (char *)__mptr - __builtin_offsetof(struct kiocb,ki_list) );}); } extern unsigned long aio_nr; extern unsigned long aio_max_nr; extern void arch_pick_mmap_layout(struct mm_struct *mm); extern unsigned long arch_get_unmapped_area(struct file *, unsigned long, unsigned long, unsigned long, unsigned long); extern unsigned long arch_get_unmapped_area_topdown(struct file *filp, unsigned long addr, unsigned long len, unsigned long pgoff, unsigned long flags); extern void arch_unmap_area(struct mm_struct *, unsigned long); extern void arch_unmap_area_topdown(struct mm_struct *, unsigned long); extern void set_dumpable(struct mm_struct *mm, int value); extern int get_dumpable(struct mm_struct *mm); struct sighand_struct { atomic_t count; struct k_sigaction action[64]; spinlock_t siglock; wait_queue_head_t signalfd_wqh; }; struct pacct_struct { int ac_flag; long ac_exitcode; unsigned long ac_mem; cputime_t ac_utime, ac_stime; unsigned long ac_minflt, ac_majflt; }; struct cpu_itimer { cputime_t expires; cputime_t incr; u32 error; u32 incr_error; }; struct task_cputime { cputime_t utime; cputime_t stime; unsigned long long sum_exec_runtime; }; struct thread_group_cputimer { struct task_cputime cputime; int running; spinlock_t lock; }; struct autogroup; struct signal_struct { atomic_t sigcnt; atomic_t live; int nr_threads; wait_queue_head_t wait_chldexit; struct task_struct *curr_target; struct sigpending shared_pending; int group_exit_code; int notify_count; struct task_struct *group_exit_task; int group_stop_count; unsigned int flags; struct list_head posix_timers; struct hrtimer real_timer; struct pid *leader_pid; ktime_t it_real_incr; struct cpu_itimer it[2]; struct thread_group_cputimer cputimer; struct task_cputime cputime_expires; struct list_head cpu_timers[3]; struct pid *tty_old_pgrp; int leader; struct tty_struct *tty; struct autogroup *autogroup; cputime_t utime, stime, cutime, cstime; cputime_t gtime; cputime_t cgtime; cputime_t prev_utime, prev_stime; unsigned long nvcsw, nivcsw, cnvcsw, cnivcsw; unsigned long min_flt, maj_flt, cmin_flt, cmaj_flt; unsigned long inblock, oublock, cinblock, coublock; unsigned long maxrss, cmaxrss; struct task_io_accounting ioac; unsigned long long sum_sched_runtime; struct rlimit rlim[16]; struct pacct_struct pacct; struct taskstats *stats; unsigned audit_tty; struct tty_audit_buf *tty_audit_buf; struct rw_semaphore threadgroup_fork_lock; int oom_adj; int oom_score_adj; int oom_score_adj_min; struct mutex cred_guard_mutex; }; static inline __attribute__((always_inline)) int signal_group_exit(const struct signal_struct *sig) { return (sig->flags & 0x00000004) || (sig->group_exit_task != ((void *)0)); } struct user_struct { atomic_t __count; atomic_t processes; atomic_t files; atomic_t sigpending; atomic_t inotify_watches; atomic_t inotify_devs; atomic_t fanotify_listeners; atomic_long_t epoll_watches; unsigned long mq_bytes; unsigned long locked_shm; struct hlist_node uidhash_node; uid_t uid; struct user_namespace *user_ns; atomic_long_t locked_vm; }; extern int uids_sysfs_init(void); extern struct user_struct *find_user(uid_t); extern struct user_struct root_user; struct backing_dev_info; struct reclaim_state; struct sched_info { unsigned long pcount; unsigned long long run_delay; unsigned long long last_arrival, last_queued; }; struct task_delay_info { spinlock_t lock; unsigned int flags; struct timespec blkio_start, blkio_end; u64 blkio_delay; u64 swapin_delay; u32 blkio_count; u32 swapin_count; struct timespec freepages_start, freepages_end; u64 freepages_delay; u32 freepages_count; }; static inline __attribute__((always_inline)) int sched_info_on(void) { return 1; } enum cpu_idle_type { CPU_IDLE, CPU_NOT_IDLE, CPU_NEWLY_IDLE, CPU_MAX_IDLE_TYPES }; enum powersavings_balance_level { POWERSAVINGS_BALANCE_NONE = 0, POWERSAVINGS_BALANCE_BASIC, POWERSAVINGS_BALANCE_WAKEUP, MAX_POWERSAVINGS_BALANCE_LEVELS }; extern int sched_mc_power_savings, sched_smt_power_savings; static inline __attribute__((always_inline)) int sd_balance_for_mc_power(void) { if (__builtin_constant_p(((sched_smt_power_savings))) ? !!((sched_smt_power_savings)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/sched.h", .line = 864, }; ______r = !!((sched_smt_power_savings)); ______f.miss_hit[______r]++; ______r; })) return 0x0100; if (__builtin_constant_p(((!sched_mc_power_savings))) ? !!((!sched_mc_power_savings)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/sched.h", .line = 867, }; ______r = !!((!sched_mc_power_savings)); ______f.miss_hit[______r]++; ______r; })) return 0x1000; return 0; } static inline __attribute__((always_inline)) int sd_balance_for_package_power(void) { if (__builtin_constant_p(((sched_mc_power_savings | sched_smt_power_savings))) ? !!((sched_mc_power_savings | sched_smt_power_savings)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/sched.h", .line = 875, }; ______r = !!((sched_mc_power_savings | sched_smt_power_savings)); ______f.miss_hit[______r]++; ______r; })) return 0x0100; return 0x1000; } extern int __attribute__((weak)) arch_sd_sibiling_asym_packing(void); static inline __attribute__((always_inline)) int sd_power_saving_flags(void) { if (__builtin_constant_p(((sched_mc_power_savings | sched_smt_power_savings))) ? !!((sched_mc_power_savings | sched_smt_power_savings)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/sched.h", .line = 891, }; ______r = !!((sched_mc_power_savings | sched_smt_power_savings)); ______f.miss_hit[______r]++; ______r; })) return 0x0002; return 0; } struct sched_group_power { atomic_t ref; unsigned int power, power_orig; }; struct sched_group { struct sched_group *next; atomic_t ref; unsigned int group_weight; struct sched_group_power *sgp; unsigned long cpumask[0]; }; static inline __attribute__((always_inline)) struct cpumask *sched_group_cpus(struct sched_group *sg) { return ((struct cpumask *)(1 ? (sg->cpumask) : (void *)sizeof(__check_is_bitmap(sg->cpumask)))); } struct sched_domain_attr { int relax_domain_level; }; extern int sched_domain_level_max; struct sched_domain { struct sched_domain *parent; struct sched_domain *child; struct sched_group *groups; unsigned long min_interval; unsigned long max_interval; unsigned int busy_factor; unsigned int imbalance_pct; unsigned int cache_nice_tries; unsigned int busy_idx; unsigned int idle_idx; unsigned int newidle_idx; unsigned int wake_idx; unsigned int forkexec_idx; unsigned int smt_gain; int flags; int level; unsigned long last_balance; unsigned int balance_interval; unsigned int nr_balance_failed; u64 last_update; unsigned int lb_count[CPU_MAX_IDLE_TYPES]; unsigned int lb_failed[CPU_MAX_IDLE_TYPES]; unsigned int lb_balanced[CPU_MAX_IDLE_TYPES]; unsigned int lb_imbalance[CPU_MAX_IDLE_TYPES]; unsigned int lb_gained[CPU_MAX_IDLE_TYPES]; unsigned int lb_hot_gained[CPU_MAX_IDLE_TYPES]; unsigned int lb_nobusyg[CPU_MAX_IDLE_TYPES]; unsigned int lb_nobusyq[CPU_MAX_IDLE_TYPES]; unsigned int alb_count; unsigned int alb_failed; unsigned int alb_pushed; unsigned int sbe_count; unsigned int sbe_balanced; unsigned int sbe_pushed; unsigned int sbf_count; unsigned int sbf_balanced; unsigned int sbf_pushed; unsigned int ttwu_wake_remote; unsigned int ttwu_move_affine; unsigned int ttwu_move_balance; char *name; union { void *private; struct rcu_head rcu; }; unsigned int span_weight; unsigned long span[0]; }; static inline __attribute__((always_inline)) struct cpumask *sched_domain_span(struct sched_domain *sd) { return ((struct cpumask *)(1 ? (sd->span) : (void *)sizeof(__check_is_bitmap(sd->span)))); } extern void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[], struct sched_domain_attr *dattr_new); cpumask_var_t *alloc_sched_domains(unsigned int ndoms); void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms); static inline __attribute__((always_inline)) int test_sd_parent(struct sched_domain *sd, int flag) { if (__builtin_constant_p(((sd->parent && (sd->parent->flags & flag)))) ? !!((sd->parent && (sd->parent->flags & flag))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/sched.h", .line = 1029, }; ______r = !!((sd->parent && (sd->parent->flags & flag))); ______f.miss_hit[______r]++; ______r; })) return 1; return 0; } unsigned long default_scale_freq_power(struct sched_domain *sd, int cpu); unsigned long default_scale_smt_power(struct sched_domain *sd, int cpu); struct io_context; static inline __attribute__((always_inline)) void prefetch_stack(struct task_struct *t) { } struct audit_context; struct mempolicy; struct pipe_inode_info; struct uts_namespace; struct rq; struct sched_domain; struct sched_class { const struct sched_class *next; void (*enqueue_task) (struct rq *rq, struct task_struct *p, int flags); void (*dequeue_task) (struct rq *rq, struct task_struct *p, int flags); void (*yield_task) (struct rq *rq); bool (*yield_to_task) (struct rq *rq, struct task_struct *p, bool preempt); void (*check_preempt_curr) (struct rq *rq, struct task_struct *p, int flags); struct task_struct * (*pick_next_task) (struct rq *rq); void (*put_prev_task) (struct rq *rq, struct task_struct *p); int (*select_task_rq)(struct task_struct *p, int sd_flag, int flags); void (*pre_schedule) (struct rq *this_rq, struct task_struct *task); void (*post_schedule) (struct rq *this_rq); void (*task_waking) (struct task_struct *task); void (*task_woken) (struct rq *this_rq, struct task_struct *task); void (*set_cpus_allowed)(struct task_struct *p, const struct cpumask *newmask); void (*rq_online)(struct rq *rq); void (*rq_offline)(struct rq *rq); void (*set_curr_task) (struct rq *rq); void (*task_tick) (struct rq *rq, struct task_struct *p, int queued); void (*task_fork) (struct task_struct *p); void (*switched_from) (struct rq *this_rq, struct task_struct *task); void (*switched_to) (struct rq *this_rq, struct task_struct *task); void (*prio_changed) (struct rq *this_rq, struct task_struct *task, int oldprio); unsigned int (*get_rr_interval) (struct rq *rq, struct task_struct *task); void (*task_move_group) (struct task_struct *p, int on_rq); }; struct load_weight { unsigned long weight, inv_weight; }; struct sched_statistics { u64 wait_start; u64 wait_max; u64 wait_count; u64 wait_sum; u64 iowait_count; u64 iowait_sum; u64 sleep_start; u64 sleep_max; s64 sum_sleep_runtime; u64 block_start; u64 block_max; u64 exec_max; u64 slice_max; u64 nr_migrations_cold; u64 nr_failed_migrations_affine; u64 nr_failed_migrations_running; u64 nr_failed_migrations_hot; u64 nr_forced_migrations; u64 nr_wakeups; u64 nr_wakeups_sync; u64 nr_wakeups_migrate; u64 nr_wakeups_local; u64 nr_wakeups_remote; u64 nr_wakeups_affine; u64 nr_wakeups_affine_attempts; u64 nr_wakeups_passive; u64 nr_wakeups_idle; }; struct sched_entity { struct load_weight load; struct rb_node run_node; struct list_head group_node; unsigned int on_rq; u64 exec_start; u64 sum_exec_runtime; u64 vruntime; u64 prev_sum_exec_runtime; u64 nr_migrations; struct sched_statistics statistics; struct sched_entity *parent; struct cfs_rq *cfs_rq; struct cfs_rq *my_q; }; struct sched_rt_entity { struct list_head run_list; unsigned long timeout; unsigned int time_slice; int nr_cpus_allowed; struct sched_rt_entity *back; }; struct rcu_node; enum perf_event_task_context { perf_invalid_context = -1, perf_hw_context = 0, perf_sw_context, perf_nr_task_contexts, }; struct task_struct { volatile long state; void *stack; atomic_t usage; unsigned int flags; unsigned int ptrace; struct task_struct *wake_entry; int on_cpu; int on_rq; int prio, static_prio, normal_prio; unsigned int rt_priority; const struct sched_class *sched_class; struct sched_entity se; struct sched_rt_entity rt; struct hlist_head preempt_notifiers; unsigned char fpu_counter; unsigned int btrace_seq; unsigned int policy; cpumask_t cpus_allowed; int rcu_read_lock_nesting; char rcu_read_unlock_special; int rcu_boosted; struct list_head rcu_node_entry; struct rcu_node *rcu_blocked_node; struct rt_mutex *rcu_boost_mutex; struct sched_info sched_info; struct list_head tasks; struct plist_node pushable_tasks; struct mm_struct *mm, *active_mm; int exit_state; int exit_code, exit_signal; int pdeath_signal; unsigned int group_stop; unsigned int personality; unsigned did_exec:1; unsigned in_execve:1; unsigned in_iowait:1; unsigned sched_reset_on_fork:1; unsigned sched_contributes_to_load:1; pid_t pid; pid_t tgid; unsigned long stack_canary; struct task_struct *real_parent; struct task_struct *parent; struct list_head children; struct list_head sibling; struct task_struct *group_leader; struct list_head ptraced; struct list_head ptrace_entry; struct pid_link pids[PIDTYPE_MAX]; struct list_head thread_group; struct completion *vfork_done; int *set_child_tid; int *clear_child_tid; cputime_t utime, stime, utimescaled, stimescaled; cputime_t gtime; cputime_t prev_utime, prev_stime; unsigned long nvcsw, nivcsw; struct timespec start_time; struct timespec real_start_time; unsigned long min_flt, maj_flt; struct task_cputime cputime_expires; struct list_head cpu_timers[3]; const struct cred *real_cred; const struct cred *cred; struct cred *replacement_session_keyring; char comm[16]; int link_count, total_link_count; struct sysv_sem sysvsem; unsigned long last_switch_count; struct thread_struct thread; struct fs_struct *fs; struct files_struct *files; struct nsproxy *nsproxy; struct signal_struct *signal; struct sighand_struct *sighand; sigset_t blocked, real_blocked; sigset_t saved_sigmask; struct sigpending pending; unsigned long sas_ss_sp; size_t sas_ss_size; int (*notifier)(void *priv); void *notifier_data; sigset_t *notifier_mask; struct audit_context *audit_context; uid_t loginuid; unsigned int sessionid; seccomp_t seccomp; u32 parent_exec_id; u32 self_exec_id; spinlock_t alloc_lock; struct irqaction *irqaction; raw_spinlock_t pi_lock; struct plist_head pi_waiters; struct rt_mutex_waiter *pi_blocked_on; struct mutex_waiter *blocked_on; unsigned int irq_events; unsigned long hardirq_enable_ip; unsigned long hardirq_disable_ip; unsigned int hardirq_enable_event; unsigned int hardirq_disable_event; int hardirqs_enabled; int hardirq_context; unsigned long softirq_disable_ip; unsigned long softirq_enable_ip; unsigned int softirq_disable_event; unsigned int softirq_enable_event; int softirqs_enabled; int softirq_context; u64 curr_chain_key; int lockdep_depth; unsigned int lockdep_recursion; struct held_lock held_locks[48UL]; gfp_t lockdep_reclaim_gfp; void *journal_info; struct bio_list *bio_list; struct blk_plug *plug; struct reclaim_state *reclaim_state; struct backing_dev_info *backing_dev_info; struct io_context *io_context; unsigned long ptrace_message; siginfo_t *last_siginfo; struct task_io_accounting ioac; u64 acct_rss_mem1; u64 acct_vm_mem1; cputime_t acct_timexpd; struct css_set *cgroups; struct list_head cg_list; struct robust_list_head *robust_list; struct list_head pi_state_list; struct futex_pi_state *pi_state_cache; struct perf_event_context *perf_event_ctxp[perf_nr_task_contexts]; struct mutex perf_event_mutex; struct list_head perf_event_list; atomic_t fs_excl; struct rcu_head rcu; struct pipe_inode_info *splice_pipe; struct task_delay_info *delays; struct prop_local_single dirties; int latency_record_count; struct latency_record latency_record[32]; unsigned long timer_slack_ns; unsigned long default_timer_slack_ns; struct list_head *scm_work_list; int curr_ret_stack; struct ftrace_ret_stack *ret_stack; unsigned long long ftrace_timestamp; atomic_t trace_overrun; atomic_t tracing_graph_pause; unsigned long trace; unsigned long trace_recursion; atomic_t ptrace_bp_refcnt; }; static inline __attribute__((always_inline)) int rt_prio(int prio) { if (__builtin_constant_p((((__builtin_constant_p(prio < 100) ? !!(prio < 100) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/sched.h", .line = 1599, }; ______r = __builtin_expect(!!(prio < 100), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(prio < 100) ? !!(prio < 100) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/sched.h", .line = 1599, }; ______r = __builtin_expect(!!(prio < 100), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/sched.h", .line = 1599, }; ______r = !!(((__builtin_constant_p(prio < 100) ? !!(prio < 100) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/sched.h", .line = 1599, }; ______r = __builtin_expect(!!(prio < 100), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) return 1; return 0; } static inline __attribute__((always_inline)) int rt_task(struct task_struct *p) { return rt_prio(p->prio); } static inline __attribute__((always_inline)) struct pid *task_pid(struct task_struct *task) { return task->pids[PIDTYPE_PID].pid; } static inline __attribute__((always_inline)) struct pid *task_tgid(struct task_struct *task) { return task->group_leader->pids[PIDTYPE_PID].pid; } static inline __attribute__((always_inline)) struct pid *task_pgrp(struct task_struct *task) { return task->group_leader->pids[PIDTYPE_PGID].pid; } static inline __attribute__((always_inline)) struct pid *task_session(struct task_struct *task) { return task->group_leader->pids[PIDTYPE_SID].pid; } struct pid_namespace; pid_t __task_pid_nr_ns(struct task_struct *task, enum pid_type type, struct pid_namespace *ns); static inline __attribute__((always_inline)) pid_t task_pid_nr(struct task_struct *tsk) { return tsk->pid; } static inline __attribute__((always_inline)) pid_t task_pid_nr_ns(struct task_struct *tsk, struct pid_namespace *ns) { return __task_pid_nr_ns(tsk, PIDTYPE_PID, ns); } static inline __attribute__((always_inline)) pid_t task_pid_vnr(struct task_struct *tsk) { return __task_pid_nr_ns(tsk, PIDTYPE_PID, ((void *)0)); } static inline __attribute__((always_inline)) pid_t task_tgid_nr(struct task_struct *tsk) { return tsk->tgid; } pid_t task_tgid_nr_ns(struct task_struct *tsk, struct pid_namespace *ns); static inline __attribute__((always_inline)) pid_t task_tgid_vnr(struct task_struct *tsk) { return pid_vnr(task_tgid(tsk)); } static inline __attribute__((always_inline)) pid_t task_pgrp_nr_ns(struct task_struct *tsk, struct pid_namespace *ns) { return __task_pid_nr_ns(tsk, PIDTYPE_PGID, ns); } static inline __attribute__((always_inline)) pid_t task_pgrp_vnr(struct task_struct *tsk) { return __task_pid_nr_ns(tsk, PIDTYPE_PGID, ((void *)0)); } static inline __attribute__((always_inline)) pid_t task_session_nr_ns(struct task_struct *tsk, struct pid_namespace *ns) { return __task_pid_nr_ns(tsk, PIDTYPE_SID, ns); } static inline __attribute__((always_inline)) pid_t task_session_vnr(struct task_struct *tsk) { return __task_pid_nr_ns(tsk, PIDTYPE_SID, ((void *)0)); } static inline __attribute__((always_inline)) pid_t task_pgrp_nr(struct task_struct *tsk) { return task_pgrp_nr_ns(tsk, &init_pid_ns); } static inline __attribute__((always_inline)) int pid_alive(struct task_struct *p) { return p->pids[PIDTYPE_PID].pid != ((void *)0); } static inline __attribute__((always_inline)) int is_global_init(struct task_struct *tsk) { return tsk->pid == 1; } extern int is_container_init(struct task_struct *tsk); extern struct pid *cad_pid; extern void free_task(struct task_struct *tsk); extern void __put_task_struct(struct task_struct *t); static inline __attribute__((always_inline)) void put_task_struct(struct task_struct *t) { if (__builtin_constant_p(((atomic_dec_and_test(&t->usage)))) ? !!((atomic_dec_and_test(&t->usage))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/sched.h", .line = 1750, }; ______r = !!((atomic_dec_and_test(&t->usage))); ______f.miss_hit[______r]++; ______r; })) __put_task_struct(t); } extern void task_times(struct task_struct *p, cputime_t *ut, cputime_t *st); extern void thread_group_times(struct task_struct *p, cputime_t *ut, cputime_t *st); extern void task_clear_group_stop_pending(struct task_struct *task); static inline __attribute__((always_inline)) void rcu_copy_process(struct task_struct *p) { p->rcu_read_lock_nesting = 0; p->rcu_read_unlock_special = 0; p->rcu_blocked_node = ((void *)0); p->rcu_boost_mutex = ((void *)0); INIT_LIST_HEAD(&p->rcu_node_entry); } extern void do_set_cpus_allowed(struct task_struct *p, const struct cpumask *new_mask); extern int set_cpus_allowed_ptr(struct task_struct *p, const struct cpumask *new_mask); static inline __attribute__((always_inline)) int set_cpus_allowed(struct task_struct *p, cpumask_t new_mask) { return set_cpus_allowed_ptr(p, &new_mask); } extern unsigned long long __attribute__((no_instrument_function)) sched_clock(void); extern u64 cpu_clock(int cpu); extern u64 local_clock(void); extern u64 sched_clock_cpu(int cpu); extern void sched_clock_init(void); extern int sched_clock_stable; extern void sched_clock_tick(void); extern void sched_clock_idle_sleep_event(void); extern void sched_clock_idle_wakeup_event(u64 delta_ns); extern void enable_sched_clock_irqtime(void); extern void disable_sched_clock_irqtime(void); extern unsigned long long task_sched_runtime(struct task_struct *task); extern unsigned long long thread_group_sched_runtime(struct task_struct *task); extern void sched_exec(void); extern void sched_clock_idle_sleep_event(void); extern void sched_clock_idle_wakeup_event(u64 delta_ns); extern void idle_task_exit(void); static inline __attribute__((always_inline)) void wake_up_idle_cpu(int cpu) { } extern unsigned int sysctl_sched_latency; extern unsigned int sysctl_sched_min_granularity; extern unsigned int sysctl_sched_wakeup_granularity; extern unsigned int sysctl_sched_child_runs_first; enum sched_tunable_scaling { SCHED_TUNABLESCALING_NONE, SCHED_TUNABLESCALING_LOG, SCHED_TUNABLESCALING_LINEAR, SCHED_TUNABLESCALING_END, }; extern enum sched_tunable_scaling sysctl_sched_tunable_scaling; extern unsigned int sysctl_sched_migration_cost; extern unsigned int sysctl_sched_nr_migrate; extern unsigned int sysctl_sched_time_avg; extern unsigned int sysctl_timer_migration; extern unsigned int sysctl_sched_shares_window; int sched_proc_update_handler(struct ctl_table *table, int write, void *buffer, size_t *length, loff_t *ppos); static inline __attribute__((always_inline)) unsigned int get_sysctl_timer_migration(void) { return sysctl_timer_migration; } extern unsigned int sysctl_sched_rt_period; extern int sysctl_sched_rt_runtime; int sched_rt_handler(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos); extern unsigned int sysctl_sched_autogroup_enabled; extern void sched_autogroup_create_attach(struct task_struct *p); extern void sched_autogroup_detach(struct task_struct *p); extern void sched_autogroup_fork(struct signal_struct *sig); extern void sched_autogroup_exit(struct signal_struct *sig); extern void proc_sched_autogroup_show_task(struct task_struct *p, struct seq_file *m); extern int proc_sched_autogroup_set_nice(struct task_struct *p, int *nice); extern int rt_mutex_getprio(struct task_struct *p); extern void rt_mutex_setprio(struct task_struct *p, int prio); extern void rt_mutex_adjust_pi(struct task_struct *p); extern bool yield_to(struct task_struct *p, bool preempt); extern void set_user_nice(struct task_struct *p, long nice); extern int task_prio(const struct task_struct *p); extern int task_nice(const struct task_struct *p); extern int can_nice(const struct task_struct *p, const int nice); extern int task_curr(const struct task_struct *p); extern int idle_cpu(int cpu); extern int sched_setscheduler(struct task_struct *, int, const struct sched_param *); extern int sched_setscheduler_nocheck(struct task_struct *, int, const struct sched_param *); extern struct task_struct *idle_task(int cpu); extern struct task_struct *curr_task(int cpu); extern void set_curr_task(int cpu, struct task_struct *p); void yield(void); extern struct exec_domain default_exec_domain; union thread_union { struct thread_info thread_info; unsigned long stack[(((1UL) << 12) << 1)/sizeof(long)]; }; static inline __attribute__((always_inline)) int kstack_end(void *addr) { return !(((unsigned long)addr+sizeof(void*)-1) & ((((1UL) << 12) << 1)-sizeof(void*))); } extern union thread_union init_thread_union; extern struct task_struct init_task; extern struct mm_struct init_mm; extern struct pid_namespace init_pid_ns; extern struct task_struct *find_task_by_vpid(pid_t nr); extern struct task_struct *find_task_by_pid_ns(pid_t nr, struct pid_namespace *ns); extern void __set_special_pids(struct pid *pid); extern struct user_struct * alloc_uid(struct user_namespace *, uid_t); static inline __attribute__((always_inline)) struct user_struct *get_uid(struct user_struct *u) { atomic_inc(&u->__count); return u; } extern void free_uid(struct user_struct *); extern void release_uids(struct user_namespace *ns); extern void xtime_update(unsigned long ticks); extern int wake_up_state(struct task_struct *tsk, unsigned int state); extern int wake_up_process(struct task_struct *tsk); extern void wake_up_new_task(struct task_struct *tsk); extern void kick_process(struct task_struct *tsk); extern void sched_fork(struct task_struct *p); extern void sched_dead(struct task_struct *p); extern void proc_caches_init(void); extern void flush_signals(struct task_struct *); extern void __flush_signals(struct task_struct *); extern void ignore_signals(struct task_struct *); extern void flush_signal_handlers(struct task_struct *, int force_default); extern int dequeue_signal(struct task_struct *tsk, sigset_t *mask, siginfo_t *info); static inline __attribute__((always_inline)) int dequeue_signal_lock(struct task_struct *tsk, sigset_t *mask, siginfo_t *info) { unsigned long flags; int ret; do { do { ({ unsigned long __dummy; typeof(flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); flags = _raw_spin_lock_irqsave(spinlock_check(&tsk->sighand->siglock)); } while (0); } while (0); ret = dequeue_signal(tsk, mask, info); spin_unlock_irqrestore(&tsk->sighand->siglock, flags); return ret; } extern void block_all_signals(int (*notifier)(void *priv), void *priv, sigset_t *mask); extern void unblock_all_signals(void); extern void release_task(struct task_struct * p); extern int send_sig_info(int, struct siginfo *, struct task_struct *); extern int force_sigsegv(int, struct task_struct *); extern int force_sig_info(int, struct siginfo *, struct task_struct *); extern int __kill_pgrp_info(int sig, struct siginfo *info, struct pid *pgrp); extern int kill_pid_info(int sig, struct siginfo *info, struct pid *pid); extern int kill_pid_info_as_uid(int, struct siginfo *, struct pid *, uid_t, uid_t, u32); extern int kill_pgrp(struct pid *pid, int sig, int priv); extern int kill_pid(struct pid *pid, int sig, int priv); extern int kill_proc_info(int, struct siginfo *, pid_t); extern int do_notify_parent(struct task_struct *, int); extern void __wake_up_parent(struct task_struct *p, struct task_struct *parent); extern void force_sig(int, struct task_struct *); extern int send_sig(int, struct task_struct *, int); extern int zap_other_threads(struct task_struct *p); extern struct sigqueue *sigqueue_alloc(void); extern void sigqueue_free(struct sigqueue *); extern int send_sigqueue(struct sigqueue *, struct task_struct *, int group); extern int do_sigaction(int, struct k_sigaction *, struct k_sigaction *); extern int do_sigaltstack(const stack_t *, stack_t *, unsigned long); static inline __attribute__((always_inline)) int kill_cad_pid(int sig, int priv) { return kill_pid(cad_pid, sig, priv); } static inline __attribute__((always_inline)) int on_sig_stack(unsigned long sp) { return sp > get_current()->sas_ss_sp && sp - get_current()->sas_ss_sp <= get_current()->sas_ss_size; } static inline __attribute__((always_inline)) int sas_ss_flags(unsigned long sp) { return (get_current()->sas_ss_size == 0 ? 2 : on_sig_stack(sp) ? 1 : 0); } extern struct mm_struct * mm_alloc(void); extern void __mmdrop(struct mm_struct *); static inline __attribute__((always_inline)) void mmdrop(struct mm_struct * mm) { if (__builtin_constant_p((((__builtin_constant_p(atomic_dec_and_test(&mm->mm_count)) ? !!(atomic_dec_and_test(&mm->mm_count)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/sched.h", .line = 2204, }; ______r = __builtin_expect(!!(atomic_dec_and_test(&mm->mm_count)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(atomic_dec_and_test(&mm->mm_count)) ? !!(atomic_dec_and_test(&mm->mm_count)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/sched.h", .line = 2204, }; ______r = __builtin_expect(!!(atomic_dec_and_test(&mm->mm_count)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/sched.h", .line = 2204, }; ______r = !!(((__builtin_constant_p(atomic_dec_and_test(&mm->mm_count)) ? !!(atomic_dec_and_test(&mm->mm_count)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/sched.h", .line = 2204, }; ______r = __builtin_expect(!!(atomic_dec_and_test(&mm->mm_count)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) __mmdrop(mm); } extern void mmput(struct mm_struct *); extern struct mm_struct *get_task_mm(struct task_struct *task); extern void mm_release(struct task_struct *, struct mm_struct *); extern struct mm_struct *dup_mm(struct task_struct *tsk); extern int copy_thread(unsigned long, unsigned long, unsigned long, struct task_struct *, struct pt_regs *); extern void flush_thread(void); extern void exit_thread(void); extern void exit_files(struct task_struct *); extern void __cleanup_sighand(struct sighand_struct *); extern void exit_itimers(struct signal_struct *); extern void flush_itimer_signals(void); extern void do_group_exit(int); extern void daemonize(const char *, ...); extern int allow_signal(int); extern int disallow_signal(int); extern int do_execve(const char *, const char * const *, const char * const *, struct pt_regs *); extern long do_fork(unsigned long, unsigned long, struct pt_regs *, unsigned long, int *, int *); struct task_struct *fork_idle(int); extern void set_task_comm(struct task_struct *tsk, char *from); extern char *get_task_comm(char *to, struct task_struct *tsk); void scheduler_ipi(void); extern unsigned long wait_task_inactive(struct task_struct *, long match_state); extern bool current_is_single_threaded(void); static inline __attribute__((always_inline)) int get_nr_threads(struct task_struct *tsk) { return tsk->signal->nr_threads; } static inline __attribute__((always_inline)) int has_group_leader_pid(struct task_struct *p) { return p->pid == p->tgid; } static inline __attribute__((always_inline)) int same_thread_group(struct task_struct *p1, struct task_struct *p2) { return p1->tgid == p2->tgid; } static inline __attribute__((always_inline)) struct task_struct *next_thread(const struct task_struct *p) { return ({typeof (*p->thread_group.next) *__ptr = (typeof (*p->thread_group.next) *)p->thread_group.next; ({ const typeof( ((struct task_struct *)0)->thread_group ) *__mptr = ((typeof(p->thread_group.next))({ typeof(*(__ptr)) *_________p1 = (typeof(*(__ptr))* )(*(volatile typeof((__ptr)) *)&((__ptr))); do { } while (0); ; do { } while (0); ((typeof(*(__ptr)) *)(_________p1)); })); (struct task_struct *)( (char *)__mptr - __builtin_offsetof(struct task_struct,thread_group) );}); }) ; } static inline __attribute__((always_inline)) int thread_group_empty(struct task_struct *p) { return list_empty(&p->thread_group); } static inline __attribute__((always_inline)) int task_detached(struct task_struct *p) { return p->exit_signal == -1; } static inline __attribute__((always_inline)) void task_lock(struct task_struct *p) { spin_lock(&p->alloc_lock); } static inline __attribute__((always_inline)) void task_unlock(struct task_struct *p) { spin_unlock(&p->alloc_lock); } extern struct sighand_struct *__lock_task_sighand(struct task_struct *tsk, unsigned long *flags); static inline __attribute__((always_inline)) void unlock_task_sighand(struct task_struct *tsk, unsigned long *flags) { spin_unlock_irqrestore(&tsk->sighand->siglock, *flags); } static inline __attribute__((always_inline)) void threadgroup_fork_read_lock(struct task_struct *tsk) { down_read(&tsk->signal->threadgroup_fork_lock); } static inline __attribute__((always_inline)) void threadgroup_fork_read_unlock(struct task_struct *tsk) { up_read(&tsk->signal->threadgroup_fork_lock); } static inline __attribute__((always_inline)) void threadgroup_fork_write_lock(struct task_struct *tsk) { down_write(&tsk->signal->threadgroup_fork_lock); } static inline __attribute__((always_inline)) void threadgroup_fork_write_unlock(struct task_struct *tsk) { up_write(&tsk->signal->threadgroup_fork_lock); } static inline __attribute__((always_inline)) void setup_thread_stack(struct task_struct *p, struct task_struct *org) { *((struct thread_info *)(p)->stack) = *((struct thread_info *)(org)->stack); ((struct thread_info *)(p)->stack)->task = p; } static inline __attribute__((always_inline)) unsigned long *end_of_stack(struct task_struct *p) { return (unsigned long *)(((struct thread_info *)(p)->stack) + 1); } static inline __attribute__((always_inline)) int object_is_on_stack(void *obj) { void *stack = ((get_current())->stack); return (obj >= stack) && (obj < (stack + (((1UL) << 12) << 1))); } extern void thread_info_cache_init(void); static inline __attribute__((always_inline)) void set_tsk_thread_flag(struct task_struct *tsk, int flag) { set_ti_thread_flag(((struct thread_info *)(tsk)->stack), flag); } static inline __attribute__((always_inline)) void clear_tsk_thread_flag(struct task_struct *tsk, int flag) { clear_ti_thread_flag(((struct thread_info *)(tsk)->stack), flag); } static inline __attribute__((always_inline)) int test_and_set_tsk_thread_flag(struct task_struct *tsk, int flag) { return test_and_set_ti_thread_flag(((struct thread_info *)(tsk)->stack), flag); } static inline __attribute__((always_inline)) int test_and_clear_tsk_thread_flag(struct task_struct *tsk, int flag) { return test_and_clear_ti_thread_flag(((struct thread_info *)(tsk)->stack), flag); } static inline __attribute__((always_inline)) int test_tsk_thread_flag(struct task_struct *tsk, int flag) { return test_ti_thread_flag(((struct thread_info *)(tsk)->stack), flag); } static inline __attribute__((always_inline)) void set_tsk_need_resched(struct task_struct *tsk) { set_tsk_thread_flag(tsk,3); } static inline __attribute__((always_inline)) void clear_tsk_need_resched(struct task_struct *tsk) { clear_tsk_thread_flag(tsk,3); } static inline __attribute__((always_inline)) int test_tsk_need_resched(struct task_struct *tsk) { return (__builtin_constant_p(test_tsk_thread_flag(tsk,3)) ? !!(test_tsk_thread_flag(tsk,3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/sched.h", .line = 2458, }; ______r = __builtin_expect(!!(test_tsk_thread_flag(tsk,3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })); } static inline __attribute__((always_inline)) int restart_syscall(void) { set_tsk_thread_flag(get_current(), 2); return -513; } static inline __attribute__((always_inline)) int signal_pending(struct task_struct *p) { return (__builtin_constant_p(test_tsk_thread_flag(p,2)) ? !!(test_tsk_thread_flag(p,2)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/sched.h", .line = 2469, }; ______r = __builtin_expect(!!(test_tsk_thread_flag(p,2)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })); } static inline __attribute__((always_inline)) int __fatal_signal_pending(struct task_struct *p) { return (__builtin_constant_p((__builtin_constant_p(9) ? __const_sigismember((&p->pending.signal), (9)) : __gen_sigismember((&p->pending.signal), (9)))) ? !!((__builtin_constant_p(9) ? __const_sigismember((&p->pending.signal), (9)) : __gen_sigismember((&p->pending.signal), (9)))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/sched.h", .line = 2474, }; ______r = __builtin_expect(!!((__builtin_constant_p(9) ? __const_sigismember((&p->pending.signal), (9)) : __gen_sigismember((&p->pending.signal), (9)))), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })); } static inline __attribute__((always_inline)) int fatal_signal_pending(struct task_struct *p) { return signal_pending(p) && __fatal_signal_pending(p); } static inline __attribute__((always_inline)) int signal_pending_state(long state, struct task_struct *p) { if (__builtin_constant_p(((!(state & (1 | 128))))) ? !!((!(state & (1 | 128)))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/sched.h", .line = 2484, }; ______r = !!((!(state & (1 | 128)))); ______f.miss_hit[______r]++; ______r; })) return 0; if (__builtin_constant_p(((!signal_pending(p)))) ? !!((!signal_pending(p))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/sched.h", .line = 2486, }; ______r = !!((!signal_pending(p))); ______f.miss_hit[______r]++; ______r; })) return 0; return (state & 1) || __fatal_signal_pending(p); } static inline __attribute__((always_inline)) int need_resched(void) { return (__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/sched.h", .line = 2494, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })); } extern int _cond_resched(void); extern int __cond_resched_lock(spinlock_t *lock); extern int __cond_resched_softirq(void); static inline __attribute__((always_inline)) int spin_needbreak(spinlock_t *lock) { return spin_is_contended(lock); } void thread_group_cputime(struct task_struct *tsk, struct task_cputime *times); void thread_group_cputimer(struct task_struct *tsk, struct task_cputime *times); static inline __attribute__((always_inline)) void thread_group_cputime_init(struct signal_struct *sig) { do { spinlock_check(&sig->cputimer.lock); do { static struct lock_class_key __key; __raw_spin_lock_init((&(&sig->cputimer.lock)->rlock), "&(&sig->cputimer.lock)->rlock", &__key); } while (0); } while (0); } extern void recalc_sigpending_and_wake(struct task_struct *t); extern void recalc_sigpending(void); extern void signal_wake_up(struct task_struct *t, int resume_stopped); static inline __attribute__((always_inline)) unsigned int task_cpu(const struct task_struct *p) { return ((struct thread_info *)(p)->stack)->cpu; } extern void set_task_cpu(struct task_struct *p, unsigned int cpu); extern long sched_setaffinity(pid_t pid, const struct cpumask *new_mask); extern long sched_getaffinity(pid_t pid, struct cpumask *mask); extern void normalize_rt_tasks(void); extern struct task_group root_task_group; extern struct task_group *sched_create_group(struct task_group *parent); extern void sched_destroy_group(struct task_group *tg); extern void sched_move_task(struct task_struct *tsk); extern int sched_group_set_shares(struct task_group *tg, unsigned long shares); extern unsigned long sched_group_shares(struct task_group *tg); extern int task_can_switch_user(struct user_struct *up, struct task_struct *tsk); static inline __attribute__((always_inline)) void add_rchar(struct task_struct *tsk, ssize_t amt) { tsk->ioac.rchar += amt; } static inline __attribute__((always_inline)) void add_wchar(struct task_struct *tsk, ssize_t amt) { tsk->ioac.wchar += amt; } static inline __attribute__((always_inline)) void inc_syscr(struct task_struct *tsk) { tsk->ioac.syscr++; } static inline __attribute__((always_inline)) void inc_syscw(struct task_struct *tsk) { tsk->ioac.syscw++; } static inline __attribute__((always_inline)) void mm_update_next_owner(struct mm_struct *mm) { } static inline __attribute__((always_inline)) void mm_init_owner(struct mm_struct *mm, struct task_struct *p) { } static inline __attribute__((always_inline)) unsigned long task_rlimit(const struct task_struct *tsk, unsigned int limit) { return (*(volatile typeof(tsk->signal->rlim[limit].rlim_cur) *)&(tsk->signal->rlim[limit].rlim_cur)); } static inline __attribute__((always_inline)) unsigned long task_rlimit_max(const struct task_struct *tsk, unsigned int limit) { return (*(volatile typeof(tsk->signal->rlim[limit].rlim_max) *)&(tsk->signal->rlim[limit].rlim_max)); } static inline __attribute__((always_inline)) unsigned long rlimit(unsigned int limit) { return task_rlimit(get_current(), limit); } static inline __attribute__((always_inline)) unsigned long rlimit_max(unsigned int limit) { return task_rlimit_max(get_current(), limit); } struct irqaction; struct softirq_action; extern struct tracepoint __tracepoint_irq_handler_entry ; static inline __attribute__((always_inline)) void trace_irq_handler_entry (int irq, struct irqaction *action) { if (__builtin_constant_p(((static_branch(&__tracepoint_irq_handler_entry.key)))) ? !!((static_branch(&__tracepoint_irq_handler_entry.key))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/trace/events/irq.h" , .line = 54 , }; ______r = !!((static_branch(&__tracepoint_irq_handler_entry.key))); ______f.miss_hit[______r]++; ______r; })) do { struct tracepoint_func *it_func_ptr; void *it_func; void *__data; if (__builtin_constant_p(((!(1)))) ? !!((!(1))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/trace/events/irq.h" , .line = 54 , }; ______r = !!((!(1))); ______f.miss_hit[______r]++; ______r; })) return; rcu_read_lock_sched_notrace(); it_func_ptr = ({ typeof(*((&__tracepoint_irq_handler_entry)->funcs)) *_________p1 = (typeof(*((&__tracepoint_irq_handler_entry)->funcs))* )(*(volatile typeof(((&__tracepoint_irq_handler_entry)->funcs)) *)&(((&__tracepoint_irq_handler_entry)->funcs))); do { } while (0); ; do { } while (0); ((typeof(*((&__tracepoint_irq_handler_entry)->funcs)) *)(_________p1)); }); if (__builtin_constant_p(((it_func_ptr))) ? !!((it_func_ptr)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/trace/events/irq.h" , .line = 54 , }; ______r = !!((it_func_ptr)); ______f.miss_hit[______r]++; ______r; })) { do { it_func = (it_func_ptr)->func; __data = (it_func_ptr)->data; ((void(*)(void *__data, int irq, struct irqaction *action))(it_func))(__data, irq, action); } while ((++it_func_ptr)->func); } rcu_read_unlock_sched_notrace(); } while (0); } static inline __attribute__((always_inline)) int register_trace_irq_handler_entry (void (*probe)(void *__data, int irq, struct irqaction *action), void *data) { return tracepoint_probe_register("irq_handler_entry", (void *)probe, data); } static inline __attribute__((always_inline)) int unregister_trace_irq_handler_entry (void (*probe)(void *__data, int irq, struct irqaction *action), void *data) { return tracepoint_probe_unregister("irq_handler_entry", (void *)probe, data); } static inline __attribute__((always_inline)) void check_trace_callback_type_irq_handler_entry (void (*cb)(void *__data, int irq, struct irqaction *action)) { } ; extern struct tracepoint __tracepoint_irq_handler_exit ; static inline __attribute__((always_inline)) void trace_irq_handler_exit (int irq, struct irqaction *action, int ret) { if (__builtin_constant_p(((static_branch(&__tracepoint_irq_handler_exit.key)))) ? !!((static_branch(&__tracepoint_irq_handler_exit.key))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/trace/events/irq.h" , .line = 85 , }; ______r = !!((static_branch(&__tracepoint_irq_handler_exit.key))); ______f.miss_hit[______r]++; ______r; })) do { struct tracepoint_func *it_func_ptr; void *it_func; void *__data; if (__builtin_constant_p(((!(1)))) ? !!((!(1))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/trace/events/irq.h" , .line = 85 , }; ______r = !!((!(1))); ______f.miss_hit[______r]++; ______r; })) return; rcu_read_lock_sched_notrace(); it_func_ptr = ({ typeof(*((&__tracepoint_irq_handler_exit)->funcs)) *_________p1 = (typeof(*((&__tracepoint_irq_handler_exit)->funcs))* )(*(volatile typeof(((&__tracepoint_irq_handler_exit)->funcs)) *)&(((&__tracepoint_irq_handler_exit)->funcs))); do { } while (0); ; do { } while (0); ((typeof(*((&__tracepoint_irq_handler_exit)->funcs)) *)(_________p1)); }); if (__builtin_constant_p(((it_func_ptr))) ? !!((it_func_ptr)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/trace/events/irq.h" , .line = 85 , }; ______r = !!((it_func_ptr)); ______f.miss_hit[______r]++; ______r; })) { do { it_func = (it_func_ptr)->func; __data = (it_func_ptr)->data; ((void(*)(void *__data, int irq, struct irqaction *action, int ret))(it_func))(__data, irq, action, ret); } while ((++it_func_ptr)->func); } rcu_read_unlock_sched_notrace(); } while (0); } static inline __attribute__((always_inline)) int register_trace_irq_handler_exit (void (*probe)(void *__data, int irq, struct irqaction *action, int ret), void *data) { return tracepoint_probe_register("irq_handler_exit", (void *)probe, data); } static inline __attribute__((always_inline)) int unregister_trace_irq_handler_exit (void (*probe)(void *__data, int irq, struct irqaction *action, int ret), void *data) { return tracepoint_probe_unregister("irq_handler_exit", (void *)probe, data); } static inline __attribute__((always_inline)) void check_trace_callback_type_irq_handler_exit (void (*cb)(void *__data, int irq, struct irqaction *action, int ret)) { } ; ; extern struct tracepoint __tracepoint_softirq_entry ; static inline __attribute__((always_inline)) void trace_softirq_entry (unsigned int vec_nr) { if (__builtin_constant_p(((static_branch(&__tracepoint_softirq_entry.key)))) ? !!((static_branch(&__tracepoint_softirq_entry.key))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/trace/events/irq.h" , .line = 117 , }; ______r = !!((static_branch(&__tracepoint_softirq_entry.key))); ______f.miss_hit[______r]++; ______r; })) do { struct tracepoint_func *it_func_ptr; void *it_func; void *__data; if (__builtin_constant_p(((!(1)))) ? !!((!(1))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/trace/events/irq.h" , .line = 117 , }; ______r = !!((!(1))); ______f.miss_hit[______r]++; ______r; })) return; rcu_read_lock_sched_notrace(); it_func_ptr = ({ typeof(*((&__tracepoint_softirq_entry)->funcs)) *_________p1 = (typeof(*((&__tracepoint_softirq_entry)->funcs))* )(*(volatile typeof(((&__tracepoint_softirq_entry)->funcs)) *)&(((&__tracepoint_softirq_entry)->funcs))); do { } while (0); ; do { } while (0); ((typeof(*((&__tracepoint_softirq_entry)->funcs)) *)(_________p1)); }); if (__builtin_constant_p(((it_func_ptr))) ? !!((it_func_ptr)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/trace/events/irq.h" , .line = 117 , }; ______r = !!((it_func_ptr)); ______f.miss_hit[______r]++; ______r; })) { do { it_func = (it_func_ptr)->func; __data = (it_func_ptr)->data; ((void(*)(void *__data, unsigned int vec_nr))(it_func))(__data, vec_nr); } while ((++it_func_ptr)->func); } rcu_read_unlock_sched_notrace(); } while (0); } static inline __attribute__((always_inline)) int register_trace_softirq_entry (void (*probe)(void *__data, unsigned int vec_nr), void *data) { return tracepoint_probe_register("softirq_entry", (void *)probe, data); } static inline __attribute__((always_inline)) int unregister_trace_softirq_entry (void (*probe)(void *__data, unsigned int vec_nr), void *data) { return tracepoint_probe_unregister("softirq_entry", (void *)probe, data); } static inline __attribute__((always_inline)) void check_trace_callback_type_softirq_entry (void (*cb)(void *__data, unsigned int vec_nr)) { } ; extern struct tracepoint __tracepoint_softirq_exit ; static inline __attribute__((always_inline)) void trace_softirq_exit (unsigned int vec_nr) { if (__builtin_constant_p(((static_branch(&__tracepoint_softirq_exit.key)))) ? !!((static_branch(&__tracepoint_softirq_exit.key))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/trace/events/irq.h" , .line = 131 , }; ______r = !!((static_branch(&__tracepoint_softirq_exit.key))); ______f.miss_hit[______r]++; ______r; })) do { struct tracepoint_func *it_func_ptr; void *it_func; void *__data; if (__builtin_constant_p(((!(1)))) ? !!((!(1))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/trace/events/irq.h" , .line = 131 , }; ______r = !!((!(1))); ______f.miss_hit[______r]++; ______r; })) return; rcu_read_lock_sched_notrace(); it_func_ptr = ({ typeof(*((&__tracepoint_softirq_exit)->funcs)) *_________p1 = (typeof(*((&__tracepoint_softirq_exit)->funcs))* )(*(volatile typeof(((&__tracepoint_softirq_exit)->funcs)) *)&(((&__tracepoint_softirq_exit)->funcs))); do { } while (0); ; do { } while (0); ((typeof(*((&__tracepoint_softirq_exit)->funcs)) *)(_________p1)); }); if (__builtin_constant_p(((it_func_ptr))) ? !!((it_func_ptr)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/trace/events/irq.h" , .line = 131 , }; ______r = !!((it_func_ptr)); ______f.miss_hit[______r]++; ______r; })) { do { it_func = (it_func_ptr)->func; __data = (it_func_ptr)->data; ((void(*)(void *__data, unsigned int vec_nr))(it_func))(__data, vec_nr); } while ((++it_func_ptr)->func); } rcu_read_unlock_sched_notrace(); } while (0); } static inline __attribute__((always_inline)) int register_trace_softirq_exit (void (*probe)(void *__data, unsigned int vec_nr), void *data) { return tracepoint_probe_register("softirq_exit", (void *)probe, data); } static inline __attribute__((always_inline)) int unregister_trace_softirq_exit (void (*probe)(void *__data, unsigned int vec_nr), void *data) { return tracepoint_probe_unregister("softirq_exit", (void *)probe, data); } static inline __attribute__((always_inline)) void check_trace_callback_type_softirq_exit (void (*cb)(void *__data, unsigned int vec_nr)) { } ; extern struct tracepoint __tracepoint_softirq_raise ; static inline __attribute__((always_inline)) void trace_softirq_raise (unsigned int vec_nr) { if (__builtin_constant_p(((static_branch(&__tracepoint_softirq_raise.key)))) ? !!((static_branch(&__tracepoint_softirq_raise.key))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/trace/events/irq.h" , .line = 145 , }; ______r = !!((static_branch(&__tracepoint_softirq_raise.key))); ______f.miss_hit[______r]++; ______r; })) do { struct tracepoint_func *it_func_ptr; void *it_func; void *__data; if (__builtin_constant_p(((!(1)))) ? !!((!(1))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/trace/events/irq.h" , .line = 145 , }; ______r = !!((!(1))); ______f.miss_hit[______r]++; ______r; })) return; rcu_read_lock_sched_notrace(); it_func_ptr = ({ typeof(*((&__tracepoint_softirq_raise)->funcs)) *_________p1 = (typeof(*((&__tracepoint_softirq_raise)->funcs))* )(*(volatile typeof(((&__tracepoint_softirq_raise)->funcs)) *)&(((&__tracepoint_softirq_raise)->funcs))); do { } while (0); ; do { } while (0); ((typeof(*((&__tracepoint_softirq_raise)->funcs)) *)(_________p1)); }); if (__builtin_constant_p(((it_func_ptr))) ? !!((it_func_ptr)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/trace/events/irq.h" , .line = 145 , }; ______r = !!((it_func_ptr)); ______f.miss_hit[______r]++; ______r; })) { do { it_func = (it_func_ptr)->func; __data = (it_func_ptr)->data; ((void(*)(void *__data, unsigned int vec_nr))(it_func))(__data, vec_nr); } while ((++it_func_ptr)->func); } rcu_read_unlock_sched_notrace(); } while (0); } static inline __attribute__((always_inline)) int register_trace_softirq_raise (void (*probe)(void *__data, unsigned int vec_nr), void *data) { return tracepoint_probe_register("softirq_raise", (void *)probe, data); } static inline __attribute__((always_inline)) int unregister_trace_softirq_raise (void (*probe)(void *__data, unsigned int vec_nr), void *data) { return tracepoint_probe_unregister("softirq_raise", (void *)probe, data); } static inline __attribute__((always_inline)) void check_trace_callback_type_softirq_raise (void (*cb)(void *__data, unsigned int vec_nr)) { } ; enum { IRQC_IS_HARDIRQ = 0, IRQC_IS_NESTED, }; typedef irqreturn_t (*irq_handler_t)(int, void *); struct irqaction { irq_handler_t handler; unsigned long flags; void *dev_id; struct irqaction *next; int irq; irq_handler_t thread_fn; struct task_struct *thread; unsigned long thread_flags; unsigned long thread_mask; const char *name; struct proc_dir_entry *dir; } __attribute__((__aligned__(1 << (6)))); extern irqreturn_t no_action(int cpl, void *dev_id); extern int __attribute__((warn_unused_result)) request_threaded_irq(unsigned int irq, irq_handler_t handler, irq_handler_t thread_fn, unsigned long flags, const char *name, void *dev); static inline __attribute__((always_inline)) int __attribute__((warn_unused_result)) request_irq(unsigned int irq, irq_handler_t handler, unsigned long flags, const char *name, void *dev) { return request_threaded_irq(irq, handler, ((void *)0), flags, name, dev); } extern int __attribute__((warn_unused_result)) request_any_context_irq(unsigned int irq, irq_handler_t handler, unsigned long flags, const char *name, void *dev_id); extern void exit_irq_thread(void); extern void free_irq(unsigned int, void *); struct device; extern int __attribute__((warn_unused_result)) devm_request_threaded_irq(struct device *dev, unsigned int irq, irq_handler_t handler, irq_handler_t thread_fn, unsigned long irqflags, const char *devname, void *dev_id); static inline __attribute__((always_inline)) int __attribute__((warn_unused_result)) devm_request_irq(struct device *dev, unsigned int irq, irq_handler_t handler, unsigned long irqflags, const char *devname, void *dev_id) { return devm_request_threaded_irq(dev, irq, handler, ((void *)0), irqflags, devname, dev_id); } extern void devm_free_irq(struct device *dev, unsigned int irq, void *dev_id); extern void disable_irq_nosync(unsigned int irq); extern void disable_irq(unsigned int irq); extern void enable_irq(unsigned int irq); extern void suspend_device_irqs(void); extern void resume_device_irqs(void); extern int check_wakeup_irqs(void); extern cpumask_var_t irq_default_affinity; extern int irq_set_affinity(unsigned int irq, const struct cpumask *cpumask); extern int irq_can_set_affinity(unsigned int irq); extern int irq_select_affinity(unsigned int irq); extern int irq_set_affinity_hint(unsigned int irq, const struct cpumask *m); struct irq_affinity_notify { unsigned int irq; struct kref kref; struct work_struct work; void (*notify)(struct irq_affinity_notify *, const cpumask_t *mask); void (*release)(struct kref *ref); }; extern int irq_set_affinity_notifier(unsigned int irq, struct irq_affinity_notify *notify); static inline __attribute__((always_inline)) void irq_run_affinity_notifiers(void) { flush_scheduled_work(); } static inline __attribute__((always_inline)) void disable_irq_nosync_lockdep(unsigned int irq) { disable_irq_nosync(irq); do { arch_local_irq_disable(); trace_hardirqs_off(); } while (0); } static inline __attribute__((always_inline)) void disable_irq_nosync_lockdep_irqsave(unsigned int irq, unsigned long *flags) { disable_irq_nosync(irq); do { do { ({ unsigned long __dummy; typeof(*flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); *flags = arch_local_irq_save(); } while (0); trace_hardirqs_off(); } while (0); } static inline __attribute__((always_inline)) void disable_irq_lockdep(unsigned int irq) { disable_irq(irq); do { arch_local_irq_disable(); trace_hardirqs_off(); } while (0); } static inline __attribute__((always_inline)) void enable_irq_lockdep(unsigned int irq) { do { trace_hardirqs_on(); arch_local_irq_enable(); } while (0); enable_irq(irq); } static inline __attribute__((always_inline)) void enable_irq_lockdep_irqrestore(unsigned int irq, unsigned long *flags) { do { if (__builtin_constant_p(((({ ({ unsigned long __dummy; typeof(*flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); arch_irqs_disabled_flags(*flags); })))) ? !!((({ ({ unsigned long __dummy; typeof(*flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); arch_irqs_disabled_flags(*flags); }))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/interrupt.h", .line = 333, }; ______r = !!((({ ({ unsigned long __dummy; typeof(*flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); arch_irqs_disabled_flags(*flags); }))); ______f.miss_hit[______r]++; ______r; })) { do { ({ unsigned long __dummy; typeof(*flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); arch_local_irq_restore(*flags); } while (0); trace_hardirqs_off(); } else { trace_hardirqs_on(); do { ({ unsigned long __dummy; typeof(*flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); arch_local_irq_restore(*flags); } while (0); } } while (0); enable_irq(irq); } extern int irq_set_irq_wake(unsigned int irq, unsigned int on); static inline __attribute__((always_inline)) int enable_irq_wake(unsigned int irq) { return irq_set_irq_wake(irq, 1); } static inline __attribute__((always_inline)) int disable_irq_wake(unsigned int irq) { return irq_set_irq_wake(irq, 0); } extern bool force_irqthreads; enum { HI_SOFTIRQ=0, TIMER_SOFTIRQ, NET_TX_SOFTIRQ, NET_RX_SOFTIRQ, BLOCK_SOFTIRQ, BLOCK_IOPOLL_SOFTIRQ, TASKLET_SOFTIRQ, SCHED_SOFTIRQ, HRTIMER_SOFTIRQ, RCU_SOFTIRQ, NR_SOFTIRQS }; extern char *softirq_to_name[NR_SOFTIRQS]; struct softirq_action { void (*action)(struct softirq_action *); }; __attribute__((regparm(0))) void do_softirq(void); __attribute__((regparm(0))) void __do_softirq(void); extern void open_softirq(int nr, void (*action)(struct softirq_action *)); extern void softirq_init(void); static inline __attribute__((always_inline)) void __raise_softirq_irqoff(unsigned int nr) { trace_softirq_raise(nr); do { typedef typeof(irq_stat.__softirq_pending) pto_T__; if (__builtin_constant_p(((0))) ? !!((0)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/interrupt.h", .line = 443, }; ______r = !!((0)); ______f.miss_hit[______r]++; ______r; })) { pto_T__ pto_tmp__; pto_tmp__ = ((1UL << nr)); (void)pto_tmp__; } switch (sizeof(irq_stat.__softirq_pending)) { case 1: asm("or" "b %1,""%%""fs"":" "%P" "0" : "+m" (irq_stat.__softirq_pending) : "qi" ((pto_T__)((1UL << nr)))); break; case 2: asm("or" "w %1,""%%""fs"":" "%P" "0" : "+m" (irq_stat.__softirq_pending) : "ri" ((pto_T__)((1UL << nr)))); break; case 4: asm("or" "l %1,""%%""fs"":" "%P" "0" : "+m" (irq_stat.__softirq_pending) : "ri" ((pto_T__)((1UL << nr)))); break; case 8: asm("or" "q %1,""%%""fs"":" "%P" "0" : "+m" (irq_stat.__softirq_pending) : "re" ((pto_T__)((1UL << nr)))); break; default: __bad_percpu_size(); } } while (0); } extern void raise_softirq_irqoff(unsigned int nr); extern void raise_softirq(unsigned int nr); extern __attribute__((section(".data..percpu" ""))) __typeof__(struct list_head [NR_SOFTIRQS]) softirq_work_list; extern __attribute__((section(".data..percpu" ""))) __typeof__(struct task_struct *) ksoftirqd; static inline __attribute__((always_inline)) struct task_struct *this_cpu_ksoftirqd(void) { return ({ typeof((ksoftirqd)) pscr_ret__; do { const void *__vpp_verify = (typeof(&((ksoftirqd))))((void *)0); (void)__vpp_verify; } while (0); switch(sizeof((ksoftirqd))) { case 1: pscr_ret__ = ({ typeof(((ksoftirqd))) pfo_ret__; switch (sizeof(((ksoftirqd)))) { case 1: asm("mov" "b ""%%""fs"":" "%P" "1"",%0" : "=q" (pfo_ret__) : "m"((ksoftirqd))); break; case 2: asm("mov" "w ""%%""fs"":" "%P" "1"",%0" : "=r" (pfo_ret__) : "m"((ksoftirqd))); break; case 4: asm("mov" "l ""%%""fs"":" "%P" "1"",%0" : "=r" (pfo_ret__) : "m"((ksoftirqd))); break; case 8: asm("mov" "q ""%%""fs"":" "%P" "1"",%0" : "=r" (pfo_ret__) : "m"((ksoftirqd))); break; default: __bad_percpu_size(); } pfo_ret__; });break; case 2: pscr_ret__ = ({ typeof(((ksoftirqd))) pfo_ret__; switch (sizeof(((ksoftirqd)))) { case 1: asm("mov" "b ""%%""fs"":" "%P" "1"",%0" : "=q" (pfo_ret__) : "m"((ksoftirqd))); break; case 2: asm("mov" "w ""%%""fs"":" "%P" "1"",%0" : "=r" (pfo_ret__) : "m"((ksoftirqd))); break; case 4: asm("mov" "l ""%%""fs"":" "%P" "1"",%0" : "=r" (pfo_ret__) : "m"((ksoftirqd))); break; case 8: asm("mov" "q ""%%""fs"":" "%P" "1"",%0" : "=r" (pfo_ret__) : "m"((ksoftirqd))); break; default: __bad_percpu_size(); } pfo_ret__; });break; case 4: pscr_ret__ = ({ typeof(((ksoftirqd))) pfo_ret__; switch (sizeof(((ksoftirqd)))) { case 1: asm("mov" "b ""%%""fs"":" "%P" "1"",%0" : "=q" (pfo_ret__) : "m"((ksoftirqd))); break; case 2: asm("mov" "w ""%%""fs"":" "%P" "1"",%0" : "=r" (pfo_ret__) : "m"((ksoftirqd))); break; case 4: asm("mov" "l ""%%""fs"":" "%P" "1"",%0" : "=r" (pfo_ret__) : "m"((ksoftirqd))); break; case 8: asm("mov" "q ""%%""fs"":" "%P" "1"",%0" : "=r" (pfo_ret__) : "m"((ksoftirqd))); break; default: __bad_percpu_size(); } pfo_ret__; });break; case 8: pscr_ret__ = ({ typeof((ksoftirqd)) ret__; do { add_preempt_count(1); __asm__ __volatile__("": : :"memory"); } while (0); ret__ = *({ do { const void *__vpp_verify = (typeof((&((ksoftirqd)))))((void *)0); (void)__vpp_verify; } while (0); ({ unsigned long __ptr; __asm__ ("" : "=r"(__ptr) : "0"((typeof(*(&((ksoftirqd)))) *)(&((ksoftirqd))))); (typeof((typeof(*(&((ksoftirqd)))) *)(&((ksoftirqd))))) (__ptr + (((__per_cpu_offset[debug_smp_processor_id()])))); }); }); do { do { __asm__ __volatile__("": : :"memory"); sub_preempt_count(1); } while (0); __asm__ __volatile__("": : :"memory"); do { if (__builtin_constant_p((((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/interrupt.h", .line = 462, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/interrupt.h", .line = 462, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/interrupt.h", .line = 462, }; ______r = !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/interrupt.h", .line = 462, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) preempt_schedule(); } while (0); } while (0); ret__; });break; default: __bad_size_call_parameter();break; } pscr_ret__; }); } extern void send_remote_softirq(struct call_single_data *cp, int cpu, int softirq); extern void __send_remote_softirq(struct call_single_data *cp, int cpu, int this_cpu, int softirq); struct tasklet_struct { struct tasklet_struct *next; unsigned long state; atomic_t count; void (*func)(unsigned long); unsigned long data; }; enum { TASKLET_STATE_SCHED, TASKLET_STATE_RUN }; static inline __attribute__((always_inline)) int tasklet_trylock(struct tasklet_struct *t) { return !test_and_set_bit(TASKLET_STATE_RUN, &(t)->state); } static inline __attribute__((always_inline)) void tasklet_unlock(struct tasklet_struct *t) { __asm__ __volatile__("": : :"memory"); clear_bit(TASKLET_STATE_RUN, &(t)->state); } static inline __attribute__((always_inline)) void tasklet_unlock_wait(struct tasklet_struct *t) { while ((__builtin_constant_p((TASKLET_STATE_RUN)) ? constant_test_bit((TASKLET_STATE_RUN), (&(t)->state)) : variable_test_bit((TASKLET_STATE_RUN), (&(t)->state)))) { __asm__ __volatile__("": : :"memory"); } } extern void __tasklet_schedule(struct tasklet_struct *t); static inline __attribute__((always_inline)) void tasklet_schedule(struct tasklet_struct *t) { if (__builtin_constant_p(((!test_and_set_bit(TASKLET_STATE_SCHED, &t->state)))) ? !!((!test_and_set_bit(TASKLET_STATE_SCHED, &t->state))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/interrupt.h", .line = 544, }; ______r = !!((!test_and_set_bit(TASKLET_STATE_SCHED, &t->state))); ______f.miss_hit[______r]++; ______r; })) __tasklet_schedule(t); } extern void __tasklet_hi_schedule(struct tasklet_struct *t); static inline __attribute__((always_inline)) void tasklet_hi_schedule(struct tasklet_struct *t) { if (__builtin_constant_p(((!test_and_set_bit(TASKLET_STATE_SCHED, &t->state)))) ? !!((!test_and_set_bit(TASKLET_STATE_SCHED, &t->state))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/interrupt.h", .line = 552, }; ______r = !!((!test_and_set_bit(TASKLET_STATE_SCHED, &t->state))); ______f.miss_hit[______r]++; ______r; })) __tasklet_hi_schedule(t); } extern void __tasklet_hi_schedule_first(struct tasklet_struct *t); static inline __attribute__((always_inline)) void tasklet_hi_schedule_first(struct tasklet_struct *t) { if (__builtin_constant_p(((!test_and_set_bit(TASKLET_STATE_SCHED, &t->state)))) ? !!((!test_and_set_bit(TASKLET_STATE_SCHED, &t->state))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/interrupt.h", .line = 566, }; ______r = !!((!test_and_set_bit(TASKLET_STATE_SCHED, &t->state))); ______f.miss_hit[______r]++; ______r; })) __tasklet_hi_schedule_first(t); } static inline __attribute__((always_inline)) void tasklet_disable_nosync(struct tasklet_struct *t) { atomic_inc(&t->count); __asm__ __volatile__("": : :"memory"); } static inline __attribute__((always_inline)) void tasklet_disable(struct tasklet_struct *t) { tasklet_disable_nosync(t); tasklet_unlock_wait(t); asm volatile ("661:\n\t" "lock; addl $0,0(%%esp)" "\n662:\n" ".section .altinstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "661b\n" " " ".long" " " "663f\n" " .word " "(0*32+26)" "\n" " .byte 662b-661b\n" " .byte 664f-663f\n" ".previous\n" ".section .discard,\"aw\",@progbits\n" " .byte 0xff + (664f-663f) - (662b-661b)\n" ".previous\n" ".section .altinstr_replacement, \"ax\"\n" "663:\n\t" "mfence" "\n664:\n" ".previous" : : : "memory"); } static inline __attribute__((always_inline)) void tasklet_enable(struct tasklet_struct *t) { __asm__ __volatile__("": : :"memory"); atomic_dec(&t->count); } static inline __attribute__((always_inline)) void tasklet_hi_enable(struct tasklet_struct *t) { __asm__ __volatile__("": : :"memory"); atomic_dec(&t->count); } extern void tasklet_kill(struct tasklet_struct *t); extern void tasklet_kill_immediate(struct tasklet_struct *t, unsigned int cpu); extern void tasklet_init(struct tasklet_struct *t, void (*func)(unsigned long), unsigned long data); struct tasklet_hrtimer { struct hrtimer timer; struct tasklet_struct tasklet; enum hrtimer_restart (*function)(struct hrtimer *); }; extern void tasklet_hrtimer_init(struct tasklet_hrtimer *ttimer, enum hrtimer_restart (*function)(struct hrtimer *), clockid_t which_clock, enum hrtimer_mode mode); static inline __attribute__((always_inline)) int tasklet_hrtimer_start(struct tasklet_hrtimer *ttimer, ktime_t time, const enum hrtimer_mode mode) { return hrtimer_start(&ttimer->timer, time, mode); } static inline __attribute__((always_inline)) void tasklet_hrtimer_cancel(struct tasklet_hrtimer *ttimer) { hrtimer_cancel(&ttimer->timer); tasklet_kill(&ttimer->tasklet); } extern unsigned long probe_irq_on(void); extern int probe_irq_off(unsigned long); extern unsigned int probe_irq_mask(unsigned long); extern void init_irq_proc(void); struct seq_file; int show_interrupts(struct seq_file *p, void *v); int arch_show_interrupts(struct seq_file *p, int prec); extern int early_irq_init(void); extern int arch_probe_nr_irqs(void); extern int arch_early_irq_init(void); struct cpu_usage_stat { cputime64_t user; cputime64_t nice; cputime64_t system; cputime64_t softirq; cputime64_t irq; cputime64_t idle; cputime64_t iowait; cputime64_t steal; cputime64_t guest; cputime64_t guest_nice; }; struct kernel_stat { struct cpu_usage_stat cpustat; unsigned long irqs_sum; unsigned int softirqs[NR_SOFTIRQS]; }; extern __attribute__((section(".data..percpu" ""))) __typeof__(struct kernel_stat) kstat; extern unsigned long long nr_context_switches(void); extern unsigned int kstat_irqs_cpu(unsigned int irq, int cpu); static inline __attribute__((always_inline)) void kstat_incr_softirqs_this_cpu(unsigned int irq) { do { do { const void *__vpp_verify = (typeof(&(((kstat.softirqs[irq])))))((void *)0); (void)__vpp_verify; } while (0); switch(sizeof(((kstat.softirqs[irq])))) { case 1: do { typedef typeof((((kstat.softirqs[irq])))) pao_T__; const int pao_ID__ = (__builtin_constant_p((1)) && (((1)) == 1 || ((1)) == -1)) ? ((1)) : 0; if (__builtin_constant_p(((0))) ? !!((0)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/kernel_stat.h", .line = 77, }; ______r = !!((0)); ______f.miss_hit[______r]++; ______r; })) { pao_T__ pao_tmp__; pao_tmp__ = ((1)); (void)pao_tmp__; } switch (sizeof((((kstat.softirqs[irq]))))) { case 1: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/kernel_stat.h", .line = 77, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incb ""%%""fs"":" "%P" "0" : "+m" ((((kstat.softirqs[irq]))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/kernel_stat.h", .line = 77, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decb ""%%""fs"":" "%P" "0" : "+m" ((((kstat.softirqs[irq]))))); else asm("addb %1, ""%%""fs"":" "%P" "0" : "+m" ((((kstat.softirqs[irq])))) : "qi" ((pao_T__)((1)))); break; case 2: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/kernel_stat.h", .line = 77, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incw ""%%""fs"":" "%P" "0" : "+m" ((((kstat.softirqs[irq]))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/kernel_stat.h", .line = 77, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decw ""%%""fs"":" "%P" "0" : "+m" ((((kstat.softirqs[irq]))))); else asm("addw %1, ""%%""fs"":" "%P" "0" : "+m" ((((kstat.softirqs[irq])))) : "ri" ((pao_T__)((1)))); break; case 4: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/kernel_stat.h", .line = 77, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incl ""%%""fs"":" "%P" "0" : "+m" ((((kstat.softirqs[irq]))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/kernel_stat.h", .line = 77, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decl ""%%""fs"":" "%P" "0" : "+m" ((((kstat.softirqs[irq]))))); else asm("addl %1, ""%%""fs"":" "%P" "0" : "+m" ((((kstat.softirqs[irq])))) : "ri" ((pao_T__)((1)))); break; case 8: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/kernel_stat.h", .line = 77, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incq ""%%""fs"":" "%P" "0" : "+m" ((((kstat.softirqs[irq]))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/kernel_stat.h", .line = 77, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decq ""%%""fs"":" "%P" "0" : "+m" ((((kstat.softirqs[irq]))))); else asm("addq %1, ""%%""fs"":" "%P" "0" : "+m" ((((kstat.softirqs[irq])))) : "re" ((pao_T__)((1)))); break; default: __bad_percpu_size(); } } while (0);break; case 2: do { typedef typeof((((kstat.softirqs[irq])))) pao_T__; const int pao_ID__ = (__builtin_constant_p((1)) && (((1)) == 1 || ((1)) == -1)) ? ((1)) : 0; if (__builtin_constant_p(((0))) ? !!((0)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/kernel_stat.h", .line = 77, }; ______r = !!((0)); ______f.miss_hit[______r]++; ______r; })) { pao_T__ pao_tmp__; pao_tmp__ = ((1)); (void)pao_tmp__; } switch (sizeof((((kstat.softirqs[irq]))))) { case 1: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/kernel_stat.h", .line = 77, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incb ""%%""fs"":" "%P" "0" : "+m" ((((kstat.softirqs[irq]))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/kernel_stat.h", .line = 77, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decb ""%%""fs"":" "%P" "0" : "+m" ((((kstat.softirqs[irq]))))); else asm("addb %1, ""%%""fs"":" "%P" "0" : "+m" ((((kstat.softirqs[irq])))) : "qi" ((pao_T__)((1)))); break; case 2: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/kernel_stat.h", .line = 77, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incw ""%%""fs"":" "%P" "0" : "+m" ((((kstat.softirqs[irq]))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/kernel_stat.h", .line = 77, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decw ""%%""fs"":" "%P" "0" : "+m" ((((kstat.softirqs[irq]))))); else asm("addw %1, ""%%""fs"":" "%P" "0" : "+m" ((((kstat.softirqs[irq])))) : "ri" ((pao_T__)((1)))); break; case 4: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/kernel_stat.h", .line = 77, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incl ""%%""fs"":" "%P" "0" : "+m" ((((kstat.softirqs[irq]))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/kernel_stat.h", .line = 77, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decl ""%%""fs"":" "%P" "0" : "+m" ((((kstat.softirqs[irq]))))); else asm("addl %1, ""%%""fs"":" "%P" "0" : "+m" ((((kstat.softirqs[irq])))) : "ri" ((pao_T__)((1)))); break; case 8: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/kernel_stat.h", .line = 77, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incq ""%%""fs"":" "%P" "0" : "+m" ((((kstat.softirqs[irq]))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/kernel_stat.h", .line = 77, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decq ""%%""fs"":" "%P" "0" : "+m" ((((kstat.softirqs[irq]))))); else asm("addq %1, ""%%""fs"":" "%P" "0" : "+m" ((((kstat.softirqs[irq])))) : "re" ((pao_T__)((1)))); break; default: __bad_percpu_size(); } } while (0);break; case 4: do { typedef typeof((((kstat.softirqs[irq])))) pao_T__; const int pao_ID__ = (__builtin_constant_p((1)) && (((1)) == 1 || ((1)) == -1)) ? ((1)) : 0; if (__builtin_constant_p(((0))) ? !!((0)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/kernel_stat.h", .line = 77, }; ______r = !!((0)); ______f.miss_hit[______r]++; ______r; })) { pao_T__ pao_tmp__; pao_tmp__ = ((1)); (void)pao_tmp__; } switch (sizeof((((kstat.softirqs[irq]))))) { case 1: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/kernel_stat.h", .line = 77, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incb ""%%""fs"":" "%P" "0" : "+m" ((((kstat.softirqs[irq]))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/kernel_stat.h", .line = 77, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decb ""%%""fs"":" "%P" "0" : "+m" ((((kstat.softirqs[irq]))))); else asm("addb %1, ""%%""fs"":" "%P" "0" : "+m" ((((kstat.softirqs[irq])))) : "qi" ((pao_T__)((1)))); break; case 2: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/kernel_stat.h", .line = 77, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incw ""%%""fs"":" "%P" "0" : "+m" ((((kstat.softirqs[irq]))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/kernel_stat.h", .line = 77, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decw ""%%""fs"":" "%P" "0" : "+m" ((((kstat.softirqs[irq]))))); else asm("addw %1, ""%%""fs"":" "%P" "0" : "+m" ((((kstat.softirqs[irq])))) : "ri" ((pao_T__)((1)))); break; case 4: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/kernel_stat.h", .line = 77, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incl ""%%""fs"":" "%P" "0" : "+m" ((((kstat.softirqs[irq]))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/kernel_stat.h", .line = 77, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decl ""%%""fs"":" "%P" "0" : "+m" ((((kstat.softirqs[irq]))))); else asm("addl %1, ""%%""fs"":" "%P" "0" : "+m" ((((kstat.softirqs[irq])))) : "ri" ((pao_T__)((1)))); break; case 8: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/kernel_stat.h", .line = 77, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incq ""%%""fs"":" "%P" "0" : "+m" ((((kstat.softirqs[irq]))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/kernel_stat.h", .line = 77, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decq ""%%""fs"":" "%P" "0" : "+m" ((((kstat.softirqs[irq]))))); else asm("addq %1, ""%%""fs"":" "%P" "0" : "+m" ((((kstat.softirqs[irq])))) : "re" ((pao_T__)((1)))); break; default: __bad_percpu_size(); } } while (0);break; case 8: do { *({ unsigned long tcp_ptr__; do { const void *__vpp_verify = (typeof(&((((kstat.softirqs[irq]))))))((void *)0); (void)__vpp_verify; } while (0); asm volatile("add " "%%""fs"":" "%P" "1" ", %0" : "=r" (tcp_ptr__) : "m" (this_cpu_off), "0" (&((((kstat.softirqs[irq])))))); (typeof(*(&((((kstat.softirqs[irq])))))) *)tcp_ptr__; }) += ((1)); } while (0);break; default: __bad_size_call_parameter();break; } } while (0); } static inline __attribute__((always_inline)) unsigned int kstat_softirqs_cpu(unsigned int irq, int cpu) { return (*({ do { const void *__vpp_verify = (typeof((&(kstat))))((void *)0); (void)__vpp_verify; } while (0); ({ unsigned long __ptr; __asm__ ("" : "=r"(__ptr) : "0"((typeof(*(&(kstat))) *)(&(kstat)))); (typeof((typeof(*(&(kstat))) *)(&(kstat)))) (__ptr + (((__per_cpu_offset[cpu])))); }); })).softirqs[irq]; } extern unsigned int kstat_irqs(unsigned int irq); static inline __attribute__((always_inline)) unsigned int kstat_cpu_irqs_sum(unsigned int cpu) { return (*({ do { const void *__vpp_verify = (typeof((&(kstat))))((void *)0); (void)__vpp_verify; } while (0); ({ unsigned long __ptr; __asm__ ("" : "=r"(__ptr) : "0"((typeof(*(&(kstat))) *)(&(kstat)))); (typeof((typeof(*(&(kstat))) *)(&(kstat)))) (__ptr + (((__per_cpu_offset[cpu])))); }); })).irqs_sum; } extern unsigned long long task_delta_exec(struct task_struct *); extern void account_user_time(struct task_struct *, cputime_t, cputime_t); extern void account_system_time(struct task_struct *, int, cputime_t, cputime_t); extern void account_steal_time(cputime_t); extern void account_idle_time(cputime_t); extern void account_process_tick(struct task_struct *, int user); extern void account_steal_ticks(unsigned long ticks); extern void account_idle_ticks(unsigned long ticks); struct task_struct; struct user_regset; typedef int user_regset_active_fn(struct task_struct *target, const struct user_regset *regset); typedef int user_regset_get_fn(struct task_struct *target, const struct user_regset *regset, unsigned int pos, unsigned int count, void *kbuf, void *ubuf); typedef int user_regset_set_fn(struct task_struct *target, const struct user_regset *regset, unsigned int pos, unsigned int count, const void *kbuf, const void *ubuf); typedef int user_regset_writeback_fn(struct task_struct *target, const struct user_regset *regset, int immediate); struct user_regset { user_regset_get_fn *get; user_regset_set_fn *set; user_regset_active_fn *active; user_regset_writeback_fn *writeback; unsigned int n; unsigned int size; unsigned int align; unsigned int bias; unsigned int core_note_type; }; struct user_regset_view { const char *name; const struct user_regset *regsets; unsigned int n; u32 e_flags; u16 e_machine; u8 ei_osabi; }; const struct user_regset_view *task_user_regset_view(struct task_struct *tsk); static inline __attribute__((always_inline)) int user_regset_copyout(unsigned int *pos, unsigned int *count, void **kbuf, void **ubuf, const void *data, const int start_pos, const int end_pos) { if (__builtin_constant_p(((*count == 0))) ? !!((*count == 0)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/regset.h", .line = 224, }; ______r = !!((*count == 0)); ______f.miss_hit[______r]++; ______r; })) return 0; do { if (__builtin_constant_p((((__builtin_constant_p(*pos < start_pos) ? !!(*pos < start_pos) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/regset.h", .line = 226, }; ______r = __builtin_expect(!!(*pos < start_pos), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(*pos < start_pos) ? !!(*pos < start_pos) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/regset.h", .line = 226, }; ______r = __builtin_expect(!!(*pos < start_pos), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/regset.h", .line = 226, }; ______r = !!(((__builtin_constant_p(*pos < start_pos) ? !!(*pos < start_pos) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/regset.h", .line = 226, }; ______r = __builtin_expect(!!(*pos < start_pos), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) do { asm volatile("1:\tud2\n" ".pushsection __bug_table,\"a\"\n" "2:\t.long 1b, %c0\n" "\t.word %c1, 0\n" "\t.org 2b+%c2\n" ".popsection" : : "i" ("include/linux/regset.h"), "i" (226), "i" (sizeof(struct bug_entry))); __builtin_unreachable(); } while (0); } while(0); if (__builtin_constant_p(((end_pos < 0 || *pos < end_pos))) ? !!((end_pos < 0 || *pos < end_pos)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/regset.h", .line = 227, }; ______r = !!((end_pos < 0 || *pos < end_pos)); ______f.miss_hit[______r]++; ______r; })) { unsigned int copy = (end_pos < 0 ? *count : ({ typeof(*count) _min1 = (*count); typeof(end_pos - *pos) _min2 = (end_pos - *pos); (void) (&_min1 == &_min2); _min1 < _min2 ? _min1 : _min2; })); data += *pos - start_pos; if (__builtin_constant_p(((*kbuf))) ? !!((*kbuf)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/regset.h", .line = 231, }; ______r = !!((*kbuf)); ______f.miss_hit[______r]++; ______r; })) { __builtin_memcpy(*kbuf, data, copy); *kbuf += copy; } else if (__builtin_constant_p(((__copy_to_user(*ubuf, data, copy)))) ? !!((__copy_to_user(*ubuf, data, copy))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/regset.h", .line = 234, }; ______r = !!((__copy_to_user(*ubuf, data, copy))); ______f.miss_hit[______r]++; ______r; })) return -14; else *ubuf += copy; *pos += copy; *count -= copy; } return 0; } static inline __attribute__((always_inline)) int user_regset_copyin(unsigned int *pos, unsigned int *count, const void **kbuf, const void **ubuf, void *data, const int start_pos, const int end_pos) { if (__builtin_constant_p(((*count == 0))) ? !!((*count == 0)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/regset.h", .line = 249, }; ______r = !!((*count == 0)); ______f.miss_hit[______r]++; ______r; })) return 0; do { if (__builtin_constant_p((((__builtin_constant_p(*pos < start_pos) ? !!(*pos < start_pos) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/regset.h", .line = 251, }; ______r = __builtin_expect(!!(*pos < start_pos), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(*pos < start_pos) ? !!(*pos < start_pos) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/regset.h", .line = 251, }; ______r = __builtin_expect(!!(*pos < start_pos), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/regset.h", .line = 251, }; ______r = !!(((__builtin_constant_p(*pos < start_pos) ? !!(*pos < start_pos) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/regset.h", .line = 251, }; ______r = __builtin_expect(!!(*pos < start_pos), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) do { asm volatile("1:\tud2\n" ".pushsection __bug_table,\"a\"\n" "2:\t.long 1b, %c0\n" "\t.word %c1, 0\n" "\t.org 2b+%c2\n" ".popsection" : : "i" ("include/linux/regset.h"), "i" (251), "i" (sizeof(struct bug_entry))); __builtin_unreachable(); } while (0); } while(0); if (__builtin_constant_p(((end_pos < 0 || *pos < end_pos))) ? !!((end_pos < 0 || *pos < end_pos)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/regset.h", .line = 252, }; ______r = !!((end_pos < 0 || *pos < end_pos)); ______f.miss_hit[______r]++; ______r; })) { unsigned int copy = (end_pos < 0 ? *count : ({ typeof(*count) _min1 = (*count); typeof(end_pos - *pos) _min2 = (end_pos - *pos); (void) (&_min1 == &_min2); _min1 < _min2 ? _min1 : _min2; })); data += *pos - start_pos; if (__builtin_constant_p(((*kbuf))) ? !!((*kbuf)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/regset.h", .line = 256, }; ______r = !!((*kbuf)); ______f.miss_hit[______r]++; ______r; })) { __builtin_memcpy(data, *kbuf, copy); *kbuf += copy; } else if (__builtin_constant_p(((__copy_from_user(data, *ubuf, copy)))) ? !!((__copy_from_user(data, *ubuf, copy))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/regset.h", .line = 259, }; ______r = !!((__copy_from_user(data, *ubuf, copy))); ______f.miss_hit[______r]++; ______r; })) return -14; else *ubuf += copy; *pos += copy; *count -= copy; } return 0; } static inline __attribute__((always_inline)) int user_regset_copyout_zero(unsigned int *pos, unsigned int *count, void **kbuf, void **ubuf, const int start_pos, const int end_pos) { if (__builtin_constant_p(((*count == 0))) ? !!((*count == 0)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/regset.h", .line = 279, }; ______r = !!((*count == 0)); ______f.miss_hit[______r]++; ______r; })) return 0; do { if (__builtin_constant_p((((__builtin_constant_p(*pos < start_pos) ? !!(*pos < start_pos) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/regset.h", .line = 281, }; ______r = __builtin_expect(!!(*pos < start_pos), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(*pos < start_pos) ? !!(*pos < start_pos) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/regset.h", .line = 281, }; ______r = __builtin_expect(!!(*pos < start_pos), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/regset.h", .line = 281, }; ______r = !!(((__builtin_constant_p(*pos < start_pos) ? !!(*pos < start_pos) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/regset.h", .line = 281, }; ______r = __builtin_expect(!!(*pos < start_pos), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) do { asm volatile("1:\tud2\n" ".pushsection __bug_table,\"a\"\n" "2:\t.long 1b, %c0\n" "\t.word %c1, 0\n" "\t.org 2b+%c2\n" ".popsection" : : "i" ("include/linux/regset.h"), "i" (281), "i" (sizeof(struct bug_entry))); __builtin_unreachable(); } while (0); } while(0); if (__builtin_constant_p(((end_pos < 0 || *pos < end_pos))) ? !!((end_pos < 0 || *pos < end_pos)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/regset.h", .line = 282, }; ______r = !!((end_pos < 0 || *pos < end_pos)); ______f.miss_hit[______r]++; ______r; })) { unsigned int copy = (end_pos < 0 ? *count : ({ typeof(*count) _min1 = (*count); typeof(end_pos - *pos) _min2 = (end_pos - *pos); (void) (&_min1 == &_min2); _min1 < _min2 ? _min1 : _min2; })); if (__builtin_constant_p(((*kbuf))) ? !!((*kbuf)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/regset.h", .line = 285, }; ______r = !!((*kbuf)); ______f.miss_hit[______r]++; ______r; })) { __builtin_memset(*kbuf, 0, copy); *kbuf += copy; } else if (__builtin_constant_p(((__clear_user(*ubuf, copy)))) ? !!((__clear_user(*ubuf, copy))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/regset.h", .line = 288, }; ______r = !!((__clear_user(*ubuf, copy))); ______f.miss_hit[______r]++; ______r; })) return -14; else *ubuf += copy; *pos += copy; *count -= copy; } return 0; } static inline __attribute__((always_inline)) int user_regset_copyin_ignore(unsigned int *pos, unsigned int *count, const void **kbuf, const void **ubuf, const int start_pos, const int end_pos) { if (__builtin_constant_p(((*count == 0))) ? !!((*count == 0)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/regset.h", .line = 305, }; ______r = !!((*count == 0)); ______f.miss_hit[______r]++; ______r; })) return 0; do { if (__builtin_constant_p((((__builtin_constant_p(*pos < start_pos) ? !!(*pos < start_pos) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/regset.h", .line = 307, }; ______r = __builtin_expect(!!(*pos < start_pos), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(*pos < start_pos) ? !!(*pos < start_pos) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/regset.h", .line = 307, }; ______r = __builtin_expect(!!(*pos < start_pos), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/regset.h", .line = 307, }; ______r = !!(((__builtin_constant_p(*pos < start_pos) ? !!(*pos < start_pos) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/regset.h", .line = 307, }; ______r = __builtin_expect(!!(*pos < start_pos), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) do { asm volatile("1:\tud2\n" ".pushsection __bug_table,\"a\"\n" "2:\t.long 1b, %c0\n" "\t.word %c1, 0\n" "\t.org 2b+%c2\n" ".popsection" : : "i" ("include/linux/regset.h"), "i" (307), "i" (sizeof(struct bug_entry))); __builtin_unreachable(); } while (0); } while(0); if (__builtin_constant_p(((end_pos < 0 || *pos < end_pos))) ? !!((end_pos < 0 || *pos < end_pos)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/regset.h", .line = 308, }; ______r = !!((end_pos < 0 || *pos < end_pos)); ______f.miss_hit[______r]++; ______r; })) { unsigned int copy = (end_pos < 0 ? *count : ({ typeof(*count) _min1 = (*count); typeof(end_pos - *pos) _min2 = (end_pos - *pos); (void) (&_min1 == &_min2); _min1 < _min2 ? _min1 : _min2; })); if (__builtin_constant_p(((*kbuf))) ? !!((*kbuf)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/regset.h", .line = 311, }; ______r = !!((*kbuf)); ______f.miss_hit[______r]++; ______r; })) *kbuf += copy; else *ubuf += copy; *pos += copy; *count -= copy; } return 0; } static inline __attribute__((always_inline)) int copy_regset_to_user(struct task_struct *target, const struct user_regset_view *view, unsigned int setno, unsigned int offset, unsigned int size, void *data) { const struct user_regset *regset = &view->regsets[setno]; if (__builtin_constant_p(((!((__builtin_constant_p(({ unsigned long flag, roksum; (void)0; asm("add %3,%1 ; sbb %0,%0 ; cmp %1,%4 ; sbb $0,%0" : "=&r" (flag), "=r" (roksum) : "1" (data), "g" ((long)(size)), "rm" (current_thread_info()->addr_limit.seg)); flag; }) == 0) ? !!(({ unsigned long flag, roksum; (void)0; asm("add %3,%1 ; sbb %0,%0 ; cmp %1,%4 ; sbb $0,%0" : "=&r" (flag), "=r" (roksum) : "1" (data), "g" ((long)(size)), "rm" (current_thread_info()->addr_limit.seg)); flag; }) == 0) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/regset.h", .line = 338, }; ______r = __builtin_expect(!!(({ unsigned long flag, roksum; (void)0; asm("add %3,%1 ; sbb %0,%0 ; cmp %1,%4 ; sbb $0,%0" : "=&r" (flag), "=r" (roksum) : "1" (data), "g" ((long)(size)), "rm" (current_thread_info()->addr_limit.seg)); flag; }) == 0), 1); ftrace_likely_update(&______f, ______r, 1); ______r; })))))) ? !!((!((__builtin_constant_p(({ unsigned long flag, roksum; (void)0; asm("add %3,%1 ; sbb %0,%0 ; cmp %1,%4 ; sbb $0,%0" : "=&r" (flag), "=r" (roksum) : "1" (data), "g" ((long)(size)), "rm" (current_thread_info()->addr_limit.seg)); flag; }) == 0) ? !!(({ unsigned long flag, roksum; (void)0; asm("add %3,%1 ; sbb %0,%0 ; cmp %1,%4 ; sbb $0,%0" : "=&r" (flag), "=r" (roksum) : "1" (data), "g" ((long)(size)), "rm" (current_thread_info()->addr_limit.seg)); flag; }) == 0) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/regset.h", .line = 338, }; ______r = __builtin_expect(!!(({ unsigned long flag, roksum; (void)0; asm("add %3,%1 ; sbb %0,%0 ; cmp %1,%4 ; sbb $0,%0" : "=&r" (flag), "=r" (roksum) : "1" (data), "g" ((long)(size)), "rm" (current_thread_info()->addr_limit.seg)); flag; }) == 0), 1); ftrace_likely_update(&______f, ______r, 1); ______r; }))))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/regset.h", .line = 338, }; ______r = !!((!((__builtin_constant_p(({ unsigned long flag, roksum; (void)0; asm("add %3,%1 ; sbb %0,%0 ; cmp %1,%4 ; sbb $0,%0" : "=&r" (flag), "=r" (roksum) : "1" (data), "g" ((long)(size)), "rm" (current_thread_info()->addr_limit.seg)); flag; }) == 0) ? !!(({ unsigned long flag, roksum; (void)0; asm("add %3,%1 ; sbb %0,%0 ; cmp %1,%4 ; sbb $0,%0" : "=&r" (flag), "=r" (roksum) : "1" (data), "g" ((long)(size)), "rm" (current_thread_info()->addr_limit.seg)); flag; }) == 0) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/regset.h", .line = 338, }; ______r = __builtin_expect(!!(({ unsigned long flag, roksum; (void)0; asm("add %3,%1 ; sbb %0,%0 ; cmp %1,%4 ; sbb $0,%0" : "=&r" (flag), "=r" (roksum) : "1" (data), "g" ((long)(size)), "rm" (current_thread_info()->addr_limit.seg)); flag; }) == 0), 1); ftrace_likely_update(&______f, ______r, 1); ______r; }))))); ______f.miss_hit[______r]++; ______r; })) return -5; return regset->get(target, regset, offset, size, ((void *)0), data); } static inline __attribute__((always_inline)) int copy_regset_from_user(struct task_struct *target, const struct user_regset_view *view, unsigned int setno, unsigned int offset, unsigned int size, const void *data) { const struct user_regset *regset = &view->regsets[setno]; if (__builtin_constant_p(((!((__builtin_constant_p(({ unsigned long flag, roksum; (void)0; asm("add %3,%1 ; sbb %0,%0 ; cmp %1,%4 ; sbb $0,%0" : "=&r" (flag), "=r" (roksum) : "1" (data), "g" ((long)(size)), "rm" (current_thread_info()->addr_limit.seg)); flag; }) == 0) ? !!(({ unsigned long flag, roksum; (void)0; asm("add %3,%1 ; sbb %0,%0 ; cmp %1,%4 ; sbb $0,%0" : "=&r" (flag), "=r" (roksum) : "1" (data), "g" ((long)(size)), "rm" (current_thread_info()->addr_limit.seg)); flag; }) == 0) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/regset.h", .line = 361, }; ______r = __builtin_expect(!!(({ unsigned long flag, roksum; (void)0; asm("add %3,%1 ; sbb %0,%0 ; cmp %1,%4 ; sbb $0,%0" : "=&r" (flag), "=r" (roksum) : "1" (data), "g" ((long)(size)), "rm" (current_thread_info()->addr_limit.seg)); flag; }) == 0), 1); ftrace_likely_update(&______f, ______r, 1); ______r; })))))) ? !!((!((__builtin_constant_p(({ unsigned long flag, roksum; (void)0; asm("add %3,%1 ; sbb %0,%0 ; cmp %1,%4 ; sbb $0,%0" : "=&r" (flag), "=r" (roksum) : "1" (data), "g" ((long)(size)), "rm" (current_thread_info()->addr_limit.seg)); flag; }) == 0) ? !!(({ unsigned long flag, roksum; (void)0; asm("add %3,%1 ; sbb %0,%0 ; cmp %1,%4 ; sbb $0,%0" : "=&r" (flag), "=r" (roksum) : "1" (data), "g" ((long)(size)), "rm" (current_thread_info()->addr_limit.seg)); flag; }) == 0) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/regset.h", .line = 361, }; ______r = __builtin_expect(!!(({ unsigned long flag, roksum; (void)0; asm("add %3,%1 ; sbb %0,%0 ; cmp %1,%4 ; sbb $0,%0" : "=&r" (flag), "=r" (roksum) : "1" (data), "g" ((long)(size)), "rm" (current_thread_info()->addr_limit.seg)); flag; }) == 0), 1); ftrace_likely_update(&______f, ______r, 1); ______r; }))))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/regset.h", .line = 361, }; ______r = !!((!((__builtin_constant_p(({ unsigned long flag, roksum; (void)0; asm("add %3,%1 ; sbb %0,%0 ; cmp %1,%4 ; sbb $0,%0" : "=&r" (flag), "=r" (roksum) : "1" (data), "g" ((long)(size)), "rm" (current_thread_info()->addr_limit.seg)); flag; }) == 0) ? !!(({ unsigned long flag, roksum; (void)0; asm("add %3,%1 ; sbb %0,%0 ; cmp %1,%4 ; sbb $0,%0" : "=&r" (flag), "=r" (roksum) : "1" (data), "g" ((long)(size)), "rm" (current_thread_info()->addr_limit.seg)); flag; }) == 0) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/regset.h", .line = 361, }; ______r = __builtin_expect(!!(({ unsigned long flag, roksum; (void)0; asm("add %3,%1 ; sbb %0,%0 ; cmp %1,%4 ; sbb $0,%0" : "=&r" (flag), "=r" (roksum) : "1" (data), "g" ((long)(size)), "rm" (current_thread_info()->addr_limit.seg)); flag; }) == 0), 1); ftrace_likely_update(&______f, ______r, 1); ______r; }))))); ______f.miss_hit[______r]++; ______r; })) return -5; return regset->set(target, regset, offset, size, ((void *)0), data); } extern unsigned int xstate_size; extern u64 pcntxt_mask; extern u64 xstate_fx_sw_bytes[6]; extern void xsave_init(void); extern void update_regset_xstate_info(unsigned int size, u64 xstate_mask); extern int init_fpu(struct task_struct *child); extern int check_for_xstate(struct i387_fxsave_struct *buf, void *fpstate, struct _fpx_sw_bytes *sw); static inline __attribute__((always_inline)) int fpu_xrstor_checking(struct fpu *fpu) { struct xsave_struct *fx = &fpu->state->xsave; int err; asm volatile("1: .byte " "0x0f,0xae,0x2f\n\t" "2:\n" ".section .fixup,\"ax\"\n" "3: movl $-1,%[err]\n" " jmp 2b\n" ".previous\n" " .section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "1b" "," "3b" "\n" " .previous\n" : [err] "=r" (err) : "D" (fx), "m" (*fx), "a" (-1), "d" (-1), "0" (0) : "memory"); return err; } static inline __attribute__((always_inline)) int xsave_user(struct xsave_struct *buf) { int err; err = __clear_user(&buf->xsave_hdr, sizeof(struct xsave_hdr_struct)); if (__builtin_constant_p((((__builtin_constant_p(err) ? !!(err) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/xsave.h", .line = 74, }; ______r = __builtin_expect(!!(err), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(err) ? !!(err) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/xsave.h", .line = 74, }; ______r = __builtin_expect(!!(err), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/xsave.h", .line = 74, }; ______r = !!(((__builtin_constant_p(err) ? !!(err) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/xsave.h", .line = 74, }; ______r = __builtin_expect(!!(err), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) return -14; __asm__ __volatile__("1: .byte " "0x0f,0xae,0x27\n" "2:\n" ".section .fixup,\"ax\"\n" "3: movl $-1,%[err]\n" " jmp 2b\n" ".previous\n" ".section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "1b,3b\n" ".previous" : [err] "=r" (err) : "D" (buf), "a" (-1), "d" (-1), "0" (0) : "memory"); if (__builtin_constant_p((((__builtin_constant_p(err) ? !!(err) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/xsave.h", .line = 90, }; ______r = __builtin_expect(!!(err), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })) && __clear_user(buf, xstate_size)))) ? !!(((__builtin_constant_p(err) ? !!(err) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/xsave.h", .line = 90, }; ______r = __builtin_expect(!!(err), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })) && __clear_user(buf, xstate_size))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/xsave.h", .line = 90, }; ______r = !!(((__builtin_constant_p(err) ? !!(err) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/xsave.h", .line = 90, }; ______r = __builtin_expect(!!(err), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })) && __clear_user(buf, xstate_size))); ______f.miss_hit[______r]++; ______r; })) err = -14; return err; } static inline __attribute__((always_inline)) int xrestore_user(struct xsave_struct *buf, u64 mask) { int err; struct xsave_struct *xstate = (( struct xsave_struct *)buf); u32 lmask = mask; u32 hmask = mask >> 32; __asm__ __volatile__("1: .byte " "0x0f,0xae,0x2f\n" "2:\n" ".section .fixup,\"ax\"\n" "3: movl $-1,%[err]\n" " jmp 2b\n" ".previous\n" ".section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "1b,3b\n" ".previous" : [err] "=r" (err) : "D" (xstate), "a" (lmask), "d" (hmask), "0" (0) : "memory"); return err; } static inline __attribute__((always_inline)) void xrstor_state(struct xsave_struct *fx, u64 mask) { u32 lmask = mask; u32 hmask = mask >> 32; asm volatile(".byte " "0x0f,0xae,0x2f\n\t" : : "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask) : "memory"); } static inline __attribute__((always_inline)) void xsave_state(struct xsave_struct *fx, u64 mask) { u32 lmask = mask; u32 hmask = mask >> 32; asm volatile(".byte " "0x0f,0xae,0x27\n\t" : : "D" (fx), "m" (*fx), "a" (lmask), "d" (hmask) : "memory"); } static inline __attribute__((always_inline)) void fpu_xsave(struct fpu *fpu) { asm volatile ("661:\n\t" ".byte " "0x0f,0xae,0x27" "\n662:\n" ".section .altinstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "661b\n" " " ".long" " " "663f\n" " .word " "(7*32+ 4)" "\n" " .byte 662b-661b\n" " .byte 664f-663f\n" ".previous\n" ".section .discard,\"aw\",@progbits\n" " .byte 0xff + (664f-663f) - (662b-661b)\n" ".previous\n" ".section .altinstr_replacement, \"ax\"\n" "663:\n\t" ".byte " "0x0f,0xae,0x37" "\n664:\n" ".previous" : : "i" (0), [fx] "D" (&fpu->state->xsave), "a" (-1), "d" (-1) : "memory") ; } extern unsigned int sig_xstate_size; extern void fpu_init(void); extern void mxcsr_feature_mask_init(void); extern int init_fpu(struct task_struct *child); extern __attribute__((regparm(0))) void math_state_restore(void); extern void __math_state_restore(void); extern int dump_fpu(struct pt_regs *, struct user_i387_struct *); extern user_regset_active_fn fpregs_active, xfpregs_active; extern user_regset_get_fn fpregs_get, xfpregs_get, fpregs_soft_get, xstateregs_get; extern user_regset_set_fn fpregs_set, xfpregs_set, fpregs_soft_set, xstateregs_set; extern struct _fpx_sw_bytes fx_sw_reserved; static inline __attribute__((always_inline)) void finit_soft_fpu(struct i387_soft_struct *soft) {} static inline __attribute__((always_inline)) __attribute__((always_inline)) __attribute__((pure)) bool use_xsaveopt(void) { return ( __builtin_constant_p((__builtin_constant_p((7*32+ 4)) && ( ((((7*32+ 4))>>5)==0 && (1UL<<(((7*32+ 4))&31) & ((1<<((0*32+ 0) & 31))|0|0|(1<<((0*32+ 6) & 31))| (1<<((0*32+ 8) & 31))|0|0|(1<<((0*32+15) & 31))| 0|0))) || ((((7*32+ 4))>>5)==1 && (1UL<<(((7*32+ 4))&31) & (0|0))) || ((((7*32+ 4))>>5)==2 && (1UL<<(((7*32+ 4))&31) & 0)) || ((((7*32+ 4))>>5)==3 && (1UL<<(((7*32+ 4))&31) & (0))) || ((((7*32+ 4))>>5)==4 && (1UL<<(((7*32+ 4))&31) & 0)) || ((((7*32+ 4))>>5)==5 && (1UL<<(((7*32+ 4))&31) & 0)) || ((((7*32+ 4))>>5)==6 && (1UL<<(((7*32+ 4))&31) & 0)) || ((((7*32+ 4))>>5)==7 && (1UL<<(((7*32+ 4))&31) & 0)) || ((((7*32+ 4))>>5)==8 && (1UL<<(((7*32+ 4))&31) & 0)) || ((((7*32+ 4))>>5)==9 && (1UL<<(((7*32+ 4))&31) & 0)) ) ? 1 : (__builtin_constant_p(((7*32+ 4))) ? constant_test_bit(((7*32+ 4)), ((unsigned long *)((&boot_cpu_data)->x86_capability))) : variable_test_bit(((7*32+ 4)), ((unsigned long *)((&boot_cpu_data)->x86_capability)))))) ? (__builtin_constant_p((7*32+ 4)) && ( ((((7*32+ 4))>>5)==0 && (1UL<<(((7*32+ 4))&31) & ((1<<((0*32+ 0) & 31))|0|0|(1<<((0*32+ 6) & 31))| (1<<((0*32+ 8) & 31))|0|0|(1<<((0*32+15) & 31))| 0|0))) || ((((7*32+ 4))>>5)==1 && (1UL<<(((7*32+ 4))&31) & (0|0))) || ((((7*32+ 4))>>5)==2 && (1UL<<(((7*32+ 4))&31) & 0)) || ((((7*32+ 4))>>5)==3 && (1UL<<(((7*32+ 4))&31) & (0))) || ((((7*32+ 4))>>5)==4 && (1UL<<(((7*32+ 4))&31) & 0)) || ((((7*32+ 4))>>5)==5 && (1UL<<(((7*32+ 4))&31) & 0)) || ((((7*32+ 4))>>5)==6 && (1UL<<(((7*32+ 4))&31) & 0)) || ((((7*32+ 4))>>5)==7 && (1UL<<(((7*32+ 4))&31) & 0)) || ((((7*32+ 4))>>5)==8 && (1UL<<(((7*32+ 4))&31) & 0)) || ((((7*32+ 4))>>5)==9 && (1UL<<(((7*32+ 4))&31) & 0)) ) ? 1 : (__builtin_constant_p(((7*32+ 4))) ? constant_test_bit(((7*32+ 4)), ((unsigned long *)((&boot_cpu_data)->x86_capability))) : variable_test_bit(((7*32+ 4)), ((unsigned long *)((&boot_cpu_data)->x86_capability))))) : __builtin_constant_p((7*32+ 4)) ? __static_cpu_has((7*32+ 4)) : (__builtin_constant_p((7*32+ 4)) && ( ((((7*32+ 4))>>5)==0 && (1UL<<(((7*32+ 4))&31) & ((1<<((0*32+ 0) & 31))|0|0|(1<<((0*32+ 6) & 31))| (1<<((0*32+ 8) & 31))|0|0|(1<<((0*32+15) & 31))| 0|0))) || ((((7*32+ 4))>>5)==1 && (1UL<<(((7*32+ 4))&31) & (0|0))) || ((((7*32+ 4))>>5)==2 && (1UL<<(((7*32+ 4))&31) & 0)) || ((((7*32+ 4))>>5)==3 && (1UL<<(((7*32+ 4))&31) & (0))) || ((((7*32+ 4))>>5)==4 && (1UL<<(((7*32+ 4))&31) & 0)) || ((((7*32+ 4))>>5)==5 && (1UL<<(((7*32+ 4))&31) & 0)) || ((((7*32+ 4))>>5)==6 && (1UL<<(((7*32+ 4))&31) & 0)) || ((((7*32+ 4))>>5)==7 && (1UL<<(((7*32+ 4))&31) & 0)) || ((((7*32+ 4))>>5)==8 && (1UL<<(((7*32+ 4))&31) & 0)) || ((((7*32+ 4))>>5)==9 && (1UL<<(((7*32+ 4))&31) & 0)) ) ? 1 : (__builtin_constant_p(((7*32+ 4))) ? constant_test_bit(((7*32+ 4)), ((unsigned long *)((&boot_cpu_data)->x86_capability))) : variable_test_bit(((7*32+ 4)), ((unsigned long *)((&boot_cpu_data)->x86_capability))))) ); } static inline __attribute__((always_inline)) __attribute__((always_inline)) __attribute__((pure)) bool use_xsave(void) { return ( __builtin_constant_p((__builtin_constant_p((4*32+26)) && ( ((((4*32+26))>>5)==0 && (1UL<<(((4*32+26))&31) & ((1<<((0*32+ 0) & 31))|0|0|(1<<((0*32+ 6) & 31))| (1<<((0*32+ 8) & 31))|0|0|(1<<((0*32+15) & 31))| 0|0))) || ((((4*32+26))>>5)==1 && (1UL<<(((4*32+26))&31) & (0|0))) || ((((4*32+26))>>5)==2 && (1UL<<(((4*32+26))&31) & 0)) || ((((4*32+26))>>5)==3 && (1UL<<(((4*32+26))&31) & (0))) || ((((4*32+26))>>5)==4 && (1UL<<(((4*32+26))&31) & 0)) || ((((4*32+26))>>5)==5 && (1UL<<(((4*32+26))&31) & 0)) || ((((4*32+26))>>5)==6 && (1UL<<(((4*32+26))&31) & 0)) || ((((4*32+26))>>5)==7 && (1UL<<(((4*32+26))&31) & 0)) || ((((4*32+26))>>5)==8 && (1UL<<(((4*32+26))&31) & 0)) || ((((4*32+26))>>5)==9 && (1UL<<(((4*32+26))&31) & 0)) ) ? 1 : (__builtin_constant_p(((4*32+26))) ? constant_test_bit(((4*32+26)), ((unsigned long *)((&boot_cpu_data)->x86_capability))) : variable_test_bit(((4*32+26)), ((unsigned long *)((&boot_cpu_data)->x86_capability)))))) ? (__builtin_constant_p((4*32+26)) && ( ((((4*32+26))>>5)==0 && (1UL<<(((4*32+26))&31) & ((1<<((0*32+ 0) & 31))|0|0|(1<<((0*32+ 6) & 31))| (1<<((0*32+ 8) & 31))|0|0|(1<<((0*32+15) & 31))| 0|0))) || ((((4*32+26))>>5)==1 && (1UL<<(((4*32+26))&31) & (0|0))) || ((((4*32+26))>>5)==2 && (1UL<<(((4*32+26))&31) & 0)) || ((((4*32+26))>>5)==3 && (1UL<<(((4*32+26))&31) & (0))) || ((((4*32+26))>>5)==4 && (1UL<<(((4*32+26))&31) & 0)) || ((((4*32+26))>>5)==5 && (1UL<<(((4*32+26))&31) & 0)) || ((((4*32+26))>>5)==6 && (1UL<<(((4*32+26))&31) & 0)) || ((((4*32+26))>>5)==7 && (1UL<<(((4*32+26))&31) & 0)) || ((((4*32+26))>>5)==8 && (1UL<<(((4*32+26))&31) & 0)) || ((((4*32+26))>>5)==9 && (1UL<<(((4*32+26))&31) & 0)) ) ? 1 : (__builtin_constant_p(((4*32+26))) ? constant_test_bit(((4*32+26)), ((unsigned long *)((&boot_cpu_data)->x86_capability))) : variable_test_bit(((4*32+26)), ((unsigned long *)((&boot_cpu_data)->x86_capability))))) : __builtin_constant_p((4*32+26)) ? __static_cpu_has((4*32+26)) : (__builtin_constant_p((4*32+26)) && ( ((((4*32+26))>>5)==0 && (1UL<<(((4*32+26))&31) & ((1<<((0*32+ 0) & 31))|0|0|(1<<((0*32+ 6) & 31))| (1<<((0*32+ 8) & 31))|0|0|(1<<((0*32+15) & 31))| 0|0))) || ((((4*32+26))>>5)==1 && (1UL<<(((4*32+26))&31) & (0|0))) || ((((4*32+26))>>5)==2 && (1UL<<(((4*32+26))&31) & 0)) || ((((4*32+26))>>5)==3 && (1UL<<(((4*32+26))&31) & (0))) || ((((4*32+26))>>5)==4 && (1UL<<(((4*32+26))&31) & 0)) || ((((4*32+26))>>5)==5 && (1UL<<(((4*32+26))&31) & 0)) || ((((4*32+26))>>5)==6 && (1UL<<(((4*32+26))&31) & 0)) || ((((4*32+26))>>5)==7 && (1UL<<(((4*32+26))&31) & 0)) || ((((4*32+26))>>5)==8 && (1UL<<(((4*32+26))&31) & 0)) || ((((4*32+26))>>5)==9 && (1UL<<(((4*32+26))&31) & 0)) ) ? 1 : (__builtin_constant_p(((4*32+26))) ? constant_test_bit(((4*32+26)), ((unsigned long *)((&boot_cpu_data)->x86_capability))) : variable_test_bit(((4*32+26)), ((unsigned long *)((&boot_cpu_data)->x86_capability))))) ); } static inline __attribute__((always_inline)) __attribute__((always_inline)) __attribute__((pure)) bool use_fxsr(void) { return ( __builtin_constant_p((__builtin_constant_p((0*32+24)) && ( ((((0*32+24))>>5)==0 && (1UL<<(((0*32+24))&31) & ((1<<((0*32+ 0) & 31))|0|0|(1<<((0*32+ 6) & 31))| (1<<((0*32+ 8) & 31))|0|0|(1<<((0*32+15) & 31))| 0|0))) || ((((0*32+24))>>5)==1 && (1UL<<(((0*32+24))&31) & (0|0))) || ((((0*32+24))>>5)==2 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==3 && (1UL<<(((0*32+24))&31) & (0))) || ((((0*32+24))>>5)==4 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==5 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==6 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==7 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==8 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==9 && (1UL<<(((0*32+24))&31) & 0)) ) ? 1 : (__builtin_constant_p(((0*32+24))) ? constant_test_bit(((0*32+24)), ((unsigned long *)((&boot_cpu_data)->x86_capability))) : variable_test_bit(((0*32+24)), ((unsigned long *)((&boot_cpu_data)->x86_capability)))))) ? (__builtin_constant_p((0*32+24)) && ( ((((0*32+24))>>5)==0 && (1UL<<(((0*32+24))&31) & ((1<<((0*32+ 0) & 31))|0|0|(1<<((0*32+ 6) & 31))| (1<<((0*32+ 8) & 31))|0|0|(1<<((0*32+15) & 31))| 0|0))) || ((((0*32+24))>>5)==1 && (1UL<<(((0*32+24))&31) & (0|0))) || ((((0*32+24))>>5)==2 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==3 && (1UL<<(((0*32+24))&31) & (0))) || ((((0*32+24))>>5)==4 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==5 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==6 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==7 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==8 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==9 && (1UL<<(((0*32+24))&31) & 0)) ) ? 1 : (__builtin_constant_p(((0*32+24))) ? constant_test_bit(((0*32+24)), ((unsigned long *)((&boot_cpu_data)->x86_capability))) : variable_test_bit(((0*32+24)), ((unsigned long *)((&boot_cpu_data)->x86_capability))))) : __builtin_constant_p((0*32+24)) ? __static_cpu_has((0*32+24)) : (__builtin_constant_p((0*32+24)) && ( ((((0*32+24))>>5)==0 && (1UL<<(((0*32+24))&31) & ((1<<((0*32+ 0) & 31))|0|0|(1<<((0*32+ 6) & 31))| (1<<((0*32+ 8) & 31))|0|0|(1<<((0*32+15) & 31))| 0|0))) || ((((0*32+24))>>5)==1 && (1UL<<(((0*32+24))&31) & (0|0))) || ((((0*32+24))>>5)==2 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==3 && (1UL<<(((0*32+24))&31) & (0))) || ((((0*32+24))>>5)==4 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==5 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==6 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==7 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==8 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==9 && (1UL<<(((0*32+24))&31) & 0)) ) ? 1 : (__builtin_constant_p(((0*32+24))) ? constant_test_bit(((0*32+24)), ((unsigned long *)((&boot_cpu_data)->x86_capability))) : variable_test_bit(((0*32+24)), ((unsigned long *)((&boot_cpu_data)->x86_capability))))) ); } extern void __sanitize_i387_state(struct task_struct *); static inline __attribute__((always_inline)) void sanitize_i387_state(struct task_struct *tsk) { if (__builtin_constant_p(((!use_xsaveopt()))) ? !!((!use_xsaveopt())) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/i387.h", .line = 85, }; ______r = !!((!use_xsaveopt())); ______f.miss_hit[______r]++; ______r; })) return; __sanitize_i387_state(tsk); } static inline __attribute__((always_inline)) int fxrstor_checking(struct i387_fxsave_struct *fx) { asm volatile ("661:\n\t" "nop ; frstor %1" "\n662:\n" ".section .altinstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "661b\n" " " ".long" " " "663f\n" " .word " "(0*32+24)" "\n" " .byte 662b-661b\n" " .byte 664f-663f\n" ".previous\n" ".section .discard,\"aw\",@progbits\n" " .byte 0xff + (664f-663f) - (662b-661b)\n" ".previous\n" ".section .altinstr_replacement, \"ax\"\n" "663:\n\t" "fxrstor %1" "\n664:\n" ".previous" : : "i" (0), "m" (*fx)) ; return 0; } static inline __attribute__((always_inline)) void fpu_fxsave(struct fpu *fpu) { asm volatile("fxsave %[fx]" : [fx] "=m" (fpu->state->fxsave)); } static inline __attribute__((always_inline)) void fpu_save_init(struct fpu *fpu) { if (__builtin_constant_p(((use_xsave()))) ? !!((use_xsave())) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/i387.h", .line = 229, }; ______r = !!((use_xsave())); ______f.miss_hit[______r]++; ______r; })) { fpu_xsave(fpu); if (__builtin_constant_p(((!(fpu->state->xsave.xsave_hdr.xstate_bv & 0x1)))) ? !!((!(fpu->state->xsave.xsave_hdr.xstate_bv & 0x1))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/i387.h", .line = 235, }; ______r = !!((!(fpu->state->xsave.xsave_hdr.xstate_bv & 0x1))); ______f.miss_hit[______r]++; ______r; })) return; } else if (__builtin_constant_p(((use_fxsr()))) ? !!((use_fxsr())) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/i387.h", .line = 237, }; ______r = !!((use_fxsr())); ______f.miss_hit[______r]++; ______r; })) { fpu_fxsave(fpu); } else { asm volatile("fnsave %[fx]; fwait" : [fx] "=m" (fpu->state->fsave)); return; } if (__builtin_constant_p((((__builtin_constant_p(fpu->state->fxsave.swd & (1 << 7)) ? !!(fpu->state->fxsave.swd & (1 << 7)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/i387.h", .line = 245, }; ______r = __builtin_expect(!!(fpu->state->fxsave.swd & (1 << 7)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(fpu->state->fxsave.swd & (1 << 7)) ? !!(fpu->state->fxsave.swd & (1 << 7)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/i387.h", .line = 245, }; ______r = __builtin_expect(!!(fpu->state->fxsave.swd & (1 << 7)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/i387.h", .line = 245, }; ______r = !!(((__builtin_constant_p(fpu->state->fxsave.swd & (1 << 7)) ? !!(fpu->state->fxsave.swd & (1 << 7)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/i387.h", .line = 245, }; ______r = __builtin_expect(!!(fpu->state->fxsave.swd & (1 << 7)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) asm volatile("fnclex"); asm volatile ("661:\n\t" ".byte " "0x90,0x8d,0xb4,0x26,0x00,0x00,0x00,0x00" "\n" ".byte " "0x89,0xf6" "\n" "\n662:\n" ".section .altinstructions,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "661b\n" " " ".long" " " "663f\n" " .word " "(3*32+10)" "\n" " .byte 662b-661b\n" " .byte 664f-663f\n" ".previous\n" ".section .discard,\"aw\",@progbits\n" " .byte 0xff + (664f-663f) - (662b-661b)\n" ".previous\n" ".section .altinstr_replacement, \"ax\"\n" "663:\n\t" "emms\n\t" "fildl %P[addr]" "\n664:\n" ".previous" : : "i" (0), [addr] "m" ((__per_cpu_offset[0]))) ; } static inline __attribute__((always_inline)) void __save_init_fpu(struct task_struct *tsk) { fpu_save_init(&tsk->thread.fpu); ((struct thread_info *)(tsk)->stack)->status &= ~0x0001; } static inline __attribute__((always_inline)) int fpu_fxrstor_checking(struct fpu *fpu) { return fxrstor_checking(&fpu->state->fxsave); } static inline __attribute__((always_inline)) int fpu_restore_checking(struct fpu *fpu) { if (__builtin_constant_p(((use_xsave()))) ? !!((use_xsave())) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/i387.h", .line = 272, }; ______r = !!((use_xsave())); ______f.miss_hit[______r]++; ______r; })) return fpu_xrstor_checking(fpu); else return fpu_fxrstor_checking(fpu); } static inline __attribute__((always_inline)) int restore_fpu_checking(struct task_struct *tsk) { return fpu_restore_checking(&tsk->thread.fpu); } extern int save_i387_xstate(void *buf); extern int restore_i387_xstate(void *buf); static inline __attribute__((always_inline)) void __unlazy_fpu(struct task_struct *tsk) { if (__builtin_constant_p(((((struct thread_info *)(tsk)->stack)->status & 0x0001))) ? !!((((struct thread_info *)(tsk)->stack)->status & 0x0001)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/i387.h", .line = 291, }; ______r = !!((((struct thread_info *)(tsk)->stack)->status & 0x0001)); ______f.miss_hit[______r]++; ______r; })) { __save_init_fpu(tsk); write_cr0(read_cr0() | 0x00000008); } else tsk->fpu_counter = 0; } static inline __attribute__((always_inline)) void __clear_fpu(struct task_struct *tsk) { if (__builtin_constant_p(((((struct thread_info *)(tsk)->stack)->status & 0x0001))) ? !!((((struct thread_info *)(tsk)->stack)->status & 0x0001)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/i387.h", .line = 300, }; ______r = !!((((struct thread_info *)(tsk)->stack)->status & 0x0001)); ______f.miss_hit[______r]++; ______r; })) { asm volatile("1: fwait\n" "2:\n" " .section __ex_table,\"a\"\n" " " ".balign 4" " " "\n" " " ".long" " " "1b" "," "2b" "\n" " .previous\n"); ((struct thread_info *)(tsk)->stack)->status &= ~0x0001; write_cr0(read_cr0() | 0x00000008); } } static inline __attribute__((always_inline)) void kernel_fpu_begin(void) { struct thread_info *me = current_thread_info(); do { add_preempt_count(1); __asm__ __volatile__("": : :"memory"); } while (0); if (__builtin_constant_p(((me->status & 0x0001))) ? !!((me->status & 0x0001)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/i387.h", .line = 314, }; ______r = !!((me->status & 0x0001)); ______f.miss_hit[______r]++; ______r; })) __save_init_fpu(me->task); else clts(); } static inline __attribute__((always_inline)) void kernel_fpu_end(void) { write_cr0(read_cr0() | 0x00000008); do { do { __asm__ __volatile__("": : :"memory"); sub_preempt_count(1); } while (0); __asm__ __volatile__("": : :"memory"); do { if (__builtin_constant_p((((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/i387.h", .line = 323, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/i387.h", .line = 323, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/i387.h", .line = 323, }; ______r = !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/i387.h", .line = 323, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) preempt_schedule(); } while (0); } while (0); } static inline __attribute__((always_inline)) bool irq_fpu_usable(void) { struct pt_regs *regs; return !(((current_thread_info()->preempt_count) & ((((1UL << (10))-1) << ((0 + 8) + 8)) | (((1UL << (8))-1) << (0 + 8)) | (((1UL << (1))-1) << (((0 + 8) + 8) + 10))))) || !(regs = get_irq_regs()) || user_mode(regs) || (read_cr0() & 0x00000008); } static inline __attribute__((always_inline)) int irq_ts_save(void) { if (__builtin_constant_p(((!(((current_thread_info()->preempt_count) & ~0x10000000) != 0)))) ? !!((!(((current_thread_info()->preempt_count) & ~0x10000000) != 0))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/i387.h", .line = 348, }; ______r = !!((!(((current_thread_info()->preempt_count) & ~0x10000000) != 0))); ______f.miss_hit[______r]++; ______r; })) return 0; if (__builtin_constant_p(((read_cr0() & 0x00000008))) ? !!((read_cr0() & 0x00000008)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/i387.h", .line = 351, }; ______r = !!((read_cr0() & 0x00000008)); ______f.miss_hit[______r]++; ______r; })) { clts(); return 1; } return 0; } static inline __attribute__((always_inline)) void irq_ts_restore(int TS_state) { if (__builtin_constant_p(((TS_state))) ? !!((TS_state)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/i387.h", .line = 361, }; ______r = !!((TS_state)); ______f.miss_hit[______r]++; ______r; })) write_cr0(read_cr0() | 0x00000008); } static inline __attribute__((always_inline)) void save_init_fpu(struct task_struct *tsk) { do { add_preempt_count(1); __asm__ __volatile__("": : :"memory"); } while (0); __save_init_fpu(tsk); write_cr0(read_cr0() | 0x00000008); do { do { __asm__ __volatile__("": : :"memory"); sub_preempt_count(1); } while (0); __asm__ __volatile__("": : :"memory"); do { if (__builtin_constant_p((((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/i387.h", .line = 373, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/i387.h", .line = 373, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/i387.h", .line = 373, }; ______r = !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/i387.h", .line = 373, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) preempt_schedule(); } while (0); } while (0); } static inline __attribute__((always_inline)) void unlazy_fpu(struct task_struct *tsk) { do { add_preempt_count(1); __asm__ __volatile__("": : :"memory"); } while (0); __unlazy_fpu(tsk); do { do { __asm__ __volatile__("": : :"memory"); sub_preempt_count(1); } while (0); __asm__ __volatile__("": : :"memory"); do { if (__builtin_constant_p((((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/i387.h", .line = 380, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/i387.h", .line = 380, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/i387.h", .line = 380, }; ______r = !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/i387.h", .line = 380, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) preempt_schedule(); } while (0); } while (0); } static inline __attribute__((always_inline)) void clear_fpu(struct task_struct *tsk) { do { add_preempt_count(1); __asm__ __volatile__("": : :"memory"); } while (0); __clear_fpu(tsk); do { do { __asm__ __volatile__("": : :"memory"); sub_preempt_count(1); } while (0); __asm__ __volatile__("": : :"memory"); do { if (__builtin_constant_p((((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/i387.h", .line = 387, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/i387.h", .line = 387, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/i387.h", .line = 387, }; ______r = !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/i387.h", .line = 387, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) preempt_schedule(); } while (0); } while (0); } static inline __attribute__((always_inline)) unsigned short get_fpu_cwd(struct task_struct *tsk) { if (__builtin_constant_p((((__builtin_constant_p((0*32+24)) && ( ((((0*32+24))>>5)==0 && (1UL<<(((0*32+24))&31) & ((1<<((0*32+ 0) & 31))|0|0|(1<<((0*32+ 6) & 31))| (1<<((0*32+ 8) & 31))|0|0|(1<<((0*32+15) & 31))| 0|0))) || ((((0*32+24))>>5)==1 && (1UL<<(((0*32+24))&31) & (0|0))) || ((((0*32+24))>>5)==2 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==3 && (1UL<<(((0*32+24))&31) & (0))) || ((((0*32+24))>>5)==4 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==5 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==6 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==7 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==8 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==9 && (1UL<<(((0*32+24))&31) & 0)) ) ? 1 : (__builtin_constant_p(((0*32+24))) ? constant_test_bit(((0*32+24)), ((unsigned long *)((&boot_cpu_data)->x86_capability))) : variable_test_bit(((0*32+24)), ((unsigned long *)((&boot_cpu_data)->x86_capability)))))))) ? !!(((__builtin_constant_p((0*32+24)) && ( ((((0*32+24))>>5)==0 && (1UL<<(((0*32+24))&31) & ((1<<((0*32+ 0) & 31))|0|0|(1<<((0*32+ 6) & 31))| (1<<((0*32+ 8) & 31))|0|0|(1<<((0*32+15) & 31))| 0|0))) || ((((0*32+24))>>5)==1 && (1UL<<(((0*32+24))&31) & (0|0))) || ((((0*32+24))>>5)==2 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==3 && (1UL<<(((0*32+24))&31) & (0))) || ((((0*32+24))>>5)==4 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==5 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==6 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==7 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==8 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==9 && (1UL<<(((0*32+24))&31) & 0)) ) ? 1 : (__builtin_constant_p(((0*32+24))) ? constant_test_bit(((0*32+24)), ((unsigned long *)((&boot_cpu_data)->x86_capability))) : variable_test_bit(((0*32+24)), ((unsigned long *)((&boot_cpu_data)->x86_capability))))))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/i387.h", .line = 395, }; ______r = !!(((__builtin_constant_p((0*32+24)) && ( ((((0*32+24))>>5)==0 && (1UL<<(((0*32+24))&31) & ((1<<((0*32+ 0) & 31))|0|0|(1<<((0*32+ 6) & 31))| (1<<((0*32+ 8) & 31))|0|0|(1<<((0*32+15) & 31))| 0|0))) || ((((0*32+24))>>5)==1 && (1UL<<(((0*32+24))&31) & (0|0))) || ((((0*32+24))>>5)==2 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==3 && (1UL<<(((0*32+24))&31) & (0))) || ((((0*32+24))>>5)==4 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==5 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==6 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==7 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==8 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==9 && (1UL<<(((0*32+24))&31) & 0)) ) ? 1 : (__builtin_constant_p(((0*32+24))) ? constant_test_bit(((0*32+24)), ((unsigned long *)((&boot_cpu_data)->x86_capability))) : variable_test_bit(((0*32+24)), ((unsigned long *)((&boot_cpu_data)->x86_capability))))))); ______f.miss_hit[______r]++; ______r; })) { return tsk->thread.fpu.state->fxsave.cwd; } else { return (unsigned short)tsk->thread.fpu.state->fsave.cwd; } } static inline __attribute__((always_inline)) unsigned short get_fpu_swd(struct task_struct *tsk) { if (__builtin_constant_p((((__builtin_constant_p((0*32+24)) && ( ((((0*32+24))>>5)==0 && (1UL<<(((0*32+24))&31) & ((1<<((0*32+ 0) & 31))|0|0|(1<<((0*32+ 6) & 31))| (1<<((0*32+ 8) & 31))|0|0|(1<<((0*32+15) & 31))| 0|0))) || ((((0*32+24))>>5)==1 && (1UL<<(((0*32+24))&31) & (0|0))) || ((((0*32+24))>>5)==2 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==3 && (1UL<<(((0*32+24))&31) & (0))) || ((((0*32+24))>>5)==4 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==5 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==6 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==7 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==8 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==9 && (1UL<<(((0*32+24))&31) & 0)) ) ? 1 : (__builtin_constant_p(((0*32+24))) ? constant_test_bit(((0*32+24)), ((unsigned long *)((&boot_cpu_data)->x86_capability))) : variable_test_bit(((0*32+24)), ((unsigned long *)((&boot_cpu_data)->x86_capability)))))))) ? !!(((__builtin_constant_p((0*32+24)) && ( ((((0*32+24))>>5)==0 && (1UL<<(((0*32+24))&31) & ((1<<((0*32+ 0) & 31))|0|0|(1<<((0*32+ 6) & 31))| (1<<((0*32+ 8) & 31))|0|0|(1<<((0*32+15) & 31))| 0|0))) || ((((0*32+24))>>5)==1 && (1UL<<(((0*32+24))&31) & (0|0))) || ((((0*32+24))>>5)==2 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==3 && (1UL<<(((0*32+24))&31) & (0))) || ((((0*32+24))>>5)==4 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==5 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==6 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==7 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==8 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==9 && (1UL<<(((0*32+24))&31) & 0)) ) ? 1 : (__builtin_constant_p(((0*32+24))) ? constant_test_bit(((0*32+24)), ((unsigned long *)((&boot_cpu_data)->x86_capability))) : variable_test_bit(((0*32+24)), ((unsigned long *)((&boot_cpu_data)->x86_capability))))))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/i387.h", .line = 404, }; ______r = !!(((__builtin_constant_p((0*32+24)) && ( ((((0*32+24))>>5)==0 && (1UL<<(((0*32+24))&31) & ((1<<((0*32+ 0) & 31))|0|0|(1<<((0*32+ 6) & 31))| (1<<((0*32+ 8) & 31))|0|0|(1<<((0*32+15) & 31))| 0|0))) || ((((0*32+24))>>5)==1 && (1UL<<(((0*32+24))&31) & (0|0))) || ((((0*32+24))>>5)==2 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==3 && (1UL<<(((0*32+24))&31) & (0))) || ((((0*32+24))>>5)==4 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==5 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==6 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==7 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==8 && (1UL<<(((0*32+24))&31) & 0)) || ((((0*32+24))>>5)==9 && (1UL<<(((0*32+24))&31) & 0)) ) ? 1 : (__builtin_constant_p(((0*32+24))) ? constant_test_bit(((0*32+24)), ((unsigned long *)((&boot_cpu_data)->x86_capability))) : variable_test_bit(((0*32+24)), ((unsigned long *)((&boot_cpu_data)->x86_capability))))))); ______f.miss_hit[______r]++; ______r; })) { return tsk->thread.fpu.state->fxsave.swd; } else { return (unsigned short)tsk->thread.fpu.state->fsave.swd; } } static inline __attribute__((always_inline)) unsigned short get_fpu_mxcsr(struct task_struct *tsk) { if (__builtin_constant_p((((__builtin_constant_p((0*32+25)) && ( ((((0*32+25))>>5)==0 && (1UL<<(((0*32+25))&31) & ((1<<((0*32+ 0) & 31))|0|0|(1<<((0*32+ 6) & 31))| (1<<((0*32+ 8) & 31))|0|0|(1<<((0*32+15) & 31))| 0|0))) || ((((0*32+25))>>5)==1 && (1UL<<(((0*32+25))&31) & (0|0))) || ((((0*32+25))>>5)==2 && (1UL<<(((0*32+25))&31) & 0)) || ((((0*32+25))>>5)==3 && (1UL<<(((0*32+25))&31) & (0))) || ((((0*32+25))>>5)==4 && (1UL<<(((0*32+25))&31) & 0)) || ((((0*32+25))>>5)==5 && (1UL<<(((0*32+25))&31) & 0)) || ((((0*32+25))>>5)==6 && (1UL<<(((0*32+25))&31) & 0)) || ((((0*32+25))>>5)==7 && (1UL<<(((0*32+25))&31) & 0)) || ((((0*32+25))>>5)==8 && (1UL<<(((0*32+25))&31) & 0)) || ((((0*32+25))>>5)==9 && (1UL<<(((0*32+25))&31) & 0)) ) ? 1 : (__builtin_constant_p(((0*32+25))) ? constant_test_bit(((0*32+25)), ((unsigned long *)((&boot_cpu_data)->x86_capability))) : variable_test_bit(((0*32+25)), ((unsigned long *)((&boot_cpu_data)->x86_capability)))))))) ? !!(((__builtin_constant_p((0*32+25)) && ( ((((0*32+25))>>5)==0 && (1UL<<(((0*32+25))&31) & ((1<<((0*32+ 0) & 31))|0|0|(1<<((0*32+ 6) & 31))| (1<<((0*32+ 8) & 31))|0|0|(1<<((0*32+15) & 31))| 0|0))) || ((((0*32+25))>>5)==1 && (1UL<<(((0*32+25))&31) & (0|0))) || ((((0*32+25))>>5)==2 && (1UL<<(((0*32+25))&31) & 0)) || ((((0*32+25))>>5)==3 && (1UL<<(((0*32+25))&31) & (0))) || ((((0*32+25))>>5)==4 && (1UL<<(((0*32+25))&31) & 0)) || ((((0*32+25))>>5)==5 && (1UL<<(((0*32+25))&31) & 0)) || ((((0*32+25))>>5)==6 && (1UL<<(((0*32+25))&31) & 0)) || ((((0*32+25))>>5)==7 && (1UL<<(((0*32+25))&31) & 0)) || ((((0*32+25))>>5)==8 && (1UL<<(((0*32+25))&31) & 0)) || ((((0*32+25))>>5)==9 && (1UL<<(((0*32+25))&31) & 0)) ) ? 1 : (__builtin_constant_p(((0*32+25))) ? constant_test_bit(((0*32+25)), ((unsigned long *)((&boot_cpu_data)->x86_capability))) : variable_test_bit(((0*32+25)), ((unsigned long *)((&boot_cpu_data)->x86_capability))))))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/i387.h", .line = 413, }; ______r = !!(((__builtin_constant_p((0*32+25)) && ( ((((0*32+25))>>5)==0 && (1UL<<(((0*32+25))&31) & ((1<<((0*32+ 0) & 31))|0|0|(1<<((0*32+ 6) & 31))| (1<<((0*32+ 8) & 31))|0|0|(1<<((0*32+15) & 31))| 0|0))) || ((((0*32+25))>>5)==1 && (1UL<<(((0*32+25))&31) & (0|0))) || ((((0*32+25))>>5)==2 && (1UL<<(((0*32+25))&31) & 0)) || ((((0*32+25))>>5)==3 && (1UL<<(((0*32+25))&31) & (0))) || ((((0*32+25))>>5)==4 && (1UL<<(((0*32+25))&31) & 0)) || ((((0*32+25))>>5)==5 && (1UL<<(((0*32+25))&31) & 0)) || ((((0*32+25))>>5)==6 && (1UL<<(((0*32+25))&31) & 0)) || ((((0*32+25))>>5)==7 && (1UL<<(((0*32+25))&31) & 0)) || ((((0*32+25))>>5)==8 && (1UL<<(((0*32+25))&31) & 0)) || ((((0*32+25))>>5)==9 && (1UL<<(((0*32+25))&31) & 0)) ) ? 1 : (__builtin_constant_p(((0*32+25))) ? constant_test_bit(((0*32+25)), ((unsigned long *)((&boot_cpu_data)->x86_capability))) : variable_test_bit(((0*32+25)), ((unsigned long *)((&boot_cpu_data)->x86_capability))))))); ______f.miss_hit[______r]++; ______r; })) { return tsk->thread.fpu.state->fxsave.mxcsr; } else { return 0x1f80; } } static bool fpu_allocated(struct fpu *fpu) { return fpu->state != ((void *)0); } static inline __attribute__((always_inline)) int fpu_alloc(struct fpu *fpu) { if (__builtin_constant_p(((fpu_allocated(fpu)))) ? !!((fpu_allocated(fpu))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/i387.h", .line = 427, }; ______r = !!((fpu_allocated(fpu))); ______f.miss_hit[______r]++; ______r; })) return 0; fpu->state = kmem_cache_alloc(task_xstate_cachep, ((( gfp_t)0x10u) | (( gfp_t)0x40u) | (( gfp_t)0x80u))); if (__builtin_constant_p(((!fpu->state))) ? !!((!fpu->state)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/i387.h", .line = 430, }; ______r = !!((!fpu->state)); ______f.miss_hit[______r]++; ______r; })) return -12; ({ int __ret_warn_on = !!((unsigned long)fpu->state & 15); if (__builtin_constant_p((((__builtin_constant_p(__ret_warn_on) ? !!(__ret_warn_on) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/i387.h", .line = 432, }; ______r = __builtin_expect(!!(__ret_warn_on), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(__ret_warn_on) ? !!(__ret_warn_on) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/i387.h", .line = 432, }; ______r = __builtin_expect(!!(__ret_warn_on), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/i387.h", .line = 432, }; ______r = !!(((__builtin_constant_p(__ret_warn_on) ? !!(__ret_warn_on) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/i387.h", .line = 432, }; ______r = __builtin_expect(!!(__ret_warn_on), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) warn_slowpath_null("/data/exp/linux-3.0.4/arch/x86/include/asm/i387.h", 432); (__builtin_constant_p(__ret_warn_on) ? !!(__ret_warn_on) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/i387.h", .line = 432, }; ______r = __builtin_expect(!!(__ret_warn_on), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })); }); return 0; } static inline __attribute__((always_inline)) void fpu_free(struct fpu *fpu) { if (__builtin_constant_p(((fpu->state))) ? !!((fpu->state)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/i387.h", .line = 438, }; ______r = !!((fpu->state)); ______f.miss_hit[______r]++; ______r; })) { kmem_cache_free(task_xstate_cachep, fpu->state); fpu->state = ((void *)0); } } static inline __attribute__((always_inline)) void fpu_copy(struct fpu *dst, struct fpu *src) { __builtin_memcpy(dst->state, src->state, xstate_size); } extern void fpu_finit(struct fpu *fpu); void crypto_aes_encrypt_x86(struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src); void crypto_aes_decrypt_x86(struct crypto_aes_ctx *ctx, u8 *dst, const u8 *src); struct bio_set; struct bio; struct bio_integrity_payload; struct page; struct block_device; typedef void (bio_end_io_t) (struct bio *, int); typedef void (bio_destructor_t) (struct bio *); struct bio_vec { struct page *bv_page; unsigned int bv_len; unsigned int bv_offset; }; struct bio { sector_t bi_sector; struct bio *bi_next; struct block_device *bi_bdev; unsigned long bi_flags; unsigned long bi_rw; unsigned short bi_vcnt; unsigned short bi_idx; unsigned int bi_phys_segments; unsigned int bi_size; unsigned int bi_seg_front_size; unsigned int bi_seg_back_size; unsigned int bi_max_vecs; unsigned int bi_comp_cpu; atomic_t bi_cnt; struct bio_vec *bi_io_vec; bio_end_io_t *bi_end_io; void *bi_private; bio_destructor_t *bi_destructor; struct bio_vec bi_inline_vecs[0]; }; enum rq_flag_bits { __REQ_WRITE, __REQ_FAILFAST_DEV, __REQ_FAILFAST_TRANSPORT, __REQ_FAILFAST_DRIVER, __REQ_SYNC, __REQ_META, __REQ_DISCARD, __REQ_NOIDLE, __REQ_RAHEAD, __REQ_THROTTLED, __REQ_SORTED, __REQ_SOFTBARRIER, __REQ_FUA, __REQ_NOMERGE, __REQ_STARTED, __REQ_DONTPREP, __REQ_QUEUED, __REQ_ELVPRIV, __REQ_FAILED, __REQ_QUIET, __REQ_PREEMPT, __REQ_ALLOCED, __REQ_COPY_USER, __REQ_FLUSH, __REQ_FLUSH_SEQ, __REQ_IO_STAT, __REQ_MIXED_MERGE, __REQ_SECURE, __REQ_NR_BITS, }; struct fstrim_range { __u64 start; __u64 len; __u64 minlen; }; struct files_stat_struct { unsigned long nr_files; unsigned long nr_free_files; unsigned long max_files; }; struct inodes_stat_t { int nr_inodes; int nr_unused; int dummy[5]; }; static inline __attribute__((always_inline)) int old_valid_dev(dev_t dev) { return ((unsigned int) ((dev) >> 20)) < 256 && ((unsigned int) ((dev) & ((1U << 20) - 1))) < 256; } static inline __attribute__((always_inline)) u16 old_encode_dev(dev_t dev) { return (((unsigned int) ((dev) >> 20)) << 8) | ((unsigned int) ((dev) & ((1U << 20) - 1))); } static inline __attribute__((always_inline)) dev_t old_decode_dev(u16 val) { return ((((val >> 8) & 255) << 20) | (val & 255)); } static inline __attribute__((always_inline)) int new_valid_dev(dev_t dev) { return 1; } static inline __attribute__((always_inline)) u32 new_encode_dev(dev_t dev) { unsigned major = ((unsigned int) ((dev) >> 20)); unsigned minor = ((unsigned int) ((dev) & ((1U << 20) - 1))); return (minor & 0xff) | (major << 8) | ((minor & ~0xff) << 12); } static inline __attribute__((always_inline)) dev_t new_decode_dev(u32 dev) { unsigned major = (dev & 0xfff00) >> 8; unsigned minor = (dev & 0xff) | ((dev >> 12) & 0xfff00); return (((major) << 20) | (minor)); } static inline __attribute__((always_inline)) int huge_valid_dev(dev_t dev) { return 1; } static inline __attribute__((always_inline)) u64 huge_encode_dev(dev_t dev) { return new_encode_dev(dev); } static inline __attribute__((always_inline)) dev_t huge_decode_dev(u64 dev) { return new_decode_dev(dev); } static inline __attribute__((always_inline)) int sysv_valid_dev(dev_t dev) { return ((unsigned int) ((dev) >> 20)) < (1<<14) && ((unsigned int) ((dev) & ((1U << 20) - 1))) < (1<<18); } static inline __attribute__((always_inline)) u32 sysv_encode_dev(dev_t dev) { return ((unsigned int) ((dev) & ((1U << 20) - 1))) | (((unsigned int) ((dev) >> 20)) << 18); } static inline __attribute__((always_inline)) unsigned sysv_major(u32 dev) { return (dev >> 18) & 0x3fff; } static inline __attribute__((always_inline)) unsigned sysv_minor(u32 dev) { return dev & 0x3ffff; } static inline __attribute__((always_inline)) void bit_spin_lock(int bitnum, unsigned long *addr) { do { add_preempt_count(1); __asm__ __volatile__("": : :"memory"); } while (0); while ((__builtin_constant_p(test_and_set_bit_lock(bitnum, addr)) ? !!(test_and_set_bit_lock(bitnum, addr)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/bit_spinlock.h", .line = 25, }; ______r = __builtin_expect(!!(test_and_set_bit_lock(bitnum, addr)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))) { do { do { __asm__ __volatile__("": : :"memory"); sub_preempt_count(1); } while (0); __asm__ __volatile__("": : :"memory"); do { if (__builtin_constant_p((((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/bit_spinlock.h", .line = 26, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/bit_spinlock.h", .line = 26, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/bit_spinlock.h", .line = 26, }; ______r = !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/bit_spinlock.h", .line = 26, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) preempt_schedule(); } while (0); } while (0); do { cpu_relax(); } while ((__builtin_constant_p((bitnum)) ? constant_test_bit((bitnum), (addr)) : variable_test_bit((bitnum), (addr)))); do { add_preempt_count(1); __asm__ __volatile__("": : :"memory"); } while (0); } (void)0; } static inline __attribute__((always_inline)) int bit_spin_trylock(int bitnum, unsigned long *addr) { do { add_preempt_count(1); __asm__ __volatile__("": : :"memory"); } while (0); if (__builtin_constant_p((((__builtin_constant_p(test_and_set_bit_lock(bitnum, addr)) ? !!(test_and_set_bit_lock(bitnum, addr)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/bit_spinlock.h", .line = 43, }; ______r = __builtin_expect(!!(test_and_set_bit_lock(bitnum, addr)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(test_and_set_bit_lock(bitnum, addr)) ? !!(test_and_set_bit_lock(bitnum, addr)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/bit_spinlock.h", .line = 43, }; ______r = __builtin_expect(!!(test_and_set_bit_lock(bitnum, addr)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/bit_spinlock.h", .line = 43, }; ______r = !!(((__builtin_constant_p(test_and_set_bit_lock(bitnum, addr)) ? !!(test_and_set_bit_lock(bitnum, addr)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/bit_spinlock.h", .line = 43, }; ______r = __builtin_expect(!!(test_and_set_bit_lock(bitnum, addr)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) { do { do { __asm__ __volatile__("": : :"memory"); sub_preempt_count(1); } while (0); __asm__ __volatile__("": : :"memory"); do { if (__builtin_constant_p((((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/bit_spinlock.h", .line = 44, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/bit_spinlock.h", .line = 44, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/bit_spinlock.h", .line = 44, }; ______r = !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/bit_spinlock.h", .line = 44, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) preempt_schedule(); } while (0); } while (0); return 0; } (void)0; return 1; } static inline __attribute__((always_inline)) void bit_spin_unlock(int bitnum, unsigned long *addr) { do { if (__builtin_constant_p((((__builtin_constant_p(!(__builtin_constant_p((bitnum)) ? constant_test_bit((bitnum), (addr)) : variable_test_bit((bitnum), (addr)))) ? !!(!(__builtin_constant_p((bitnum)) ? constant_test_bit((bitnum), (addr)) : variable_test_bit((bitnum), (addr)))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/bit_spinlock.h", .line = 58, }; ______r = __builtin_expect(!!(!(__builtin_constant_p((bitnum)) ? constant_test_bit((bitnum), (addr)) : variable_test_bit((bitnum), (addr)))), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(!(__builtin_constant_p((bitnum)) ? constant_test_bit((bitnum), (addr)) : variable_test_bit((bitnum), (addr)))) ? !!(!(__builtin_constant_p((bitnum)) ? constant_test_bit((bitnum), (addr)) : variable_test_bit((bitnum), (addr)))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/bit_spinlock.h", .line = 58, }; ______r = __builtin_expect(!!(!(__builtin_constant_p((bitnum)) ? constant_test_bit((bitnum), (addr)) : variable_test_bit((bitnum), (addr)))), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/bit_spinlock.h", .line = 58, }; ______r = !!(((__builtin_constant_p(!(__builtin_constant_p((bitnum)) ? constant_test_bit((bitnum), (addr)) : variable_test_bit((bitnum), (addr)))) ? !!(!(__builtin_constant_p((bitnum)) ? constant_test_bit((bitnum), (addr)) : variable_test_bit((bitnum), (addr)))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/bit_spinlock.h", .line = 58, }; ______r = __builtin_expect(!!(!(__builtin_constant_p((bitnum)) ? constant_test_bit((bitnum), (addr)) : variable_test_bit((bitnum), (addr)))), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) do { asm volatile("1:\tud2\n" ".pushsection __bug_table,\"a\"\n" "2:\t.long 1b, %c0\n" "\t.word %c1, 0\n" "\t.org 2b+%c2\n" ".popsection" : : "i" ("include/linux/bit_spinlock.h"), "i" (58), "i" (sizeof(struct bug_entry))); __builtin_unreachable(); } while (0); } while(0); clear_bit_unlock(bitnum, addr); do { do { __asm__ __volatile__("": : :"memory"); sub_preempt_count(1); } while (0); __asm__ __volatile__("": : :"memory"); do { if (__builtin_constant_p((((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/bit_spinlock.h", .line = 63, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/bit_spinlock.h", .line = 63, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/bit_spinlock.h", .line = 63, }; ______r = !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/bit_spinlock.h", .line = 63, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) preempt_schedule(); } while (0); } while (0); (void)0; } static inline __attribute__((always_inline)) void __bit_spin_unlock(int bitnum, unsigned long *addr) { do { if (__builtin_constant_p((((__builtin_constant_p(!(__builtin_constant_p((bitnum)) ? constant_test_bit((bitnum), (addr)) : variable_test_bit((bitnum), (addr)))) ? !!(!(__builtin_constant_p((bitnum)) ? constant_test_bit((bitnum), (addr)) : variable_test_bit((bitnum), (addr)))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/bit_spinlock.h", .line = 75, }; ______r = __builtin_expect(!!(!(__builtin_constant_p((bitnum)) ? constant_test_bit((bitnum), (addr)) : variable_test_bit((bitnum), (addr)))), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(!(__builtin_constant_p((bitnum)) ? constant_test_bit((bitnum), (addr)) : variable_test_bit((bitnum), (addr)))) ? !!(!(__builtin_constant_p((bitnum)) ? constant_test_bit((bitnum), (addr)) : variable_test_bit((bitnum), (addr)))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/bit_spinlock.h", .line = 75, }; ______r = __builtin_expect(!!(!(__builtin_constant_p((bitnum)) ? constant_test_bit((bitnum), (addr)) : variable_test_bit((bitnum), (addr)))), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/bit_spinlock.h", .line = 75, }; ______r = !!(((__builtin_constant_p(!(__builtin_constant_p((bitnum)) ? constant_test_bit((bitnum), (addr)) : variable_test_bit((bitnum), (addr)))) ? !!(!(__builtin_constant_p((bitnum)) ? constant_test_bit((bitnum), (addr)) : variable_test_bit((bitnum), (addr)))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/bit_spinlock.h", .line = 75, }; ______r = __builtin_expect(!!(!(__builtin_constant_p((bitnum)) ? constant_test_bit((bitnum), (addr)) : variable_test_bit((bitnum), (addr)))), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) do { asm volatile("1:\tud2\n" ".pushsection __bug_table,\"a\"\n" "2:\t.long 1b, %c0\n" "\t.word %c1, 0\n" "\t.org 2b+%c2\n" ".popsection" : : "i" ("include/linux/bit_spinlock.h"), "i" (75), "i" (sizeof(struct bug_entry))); __builtin_unreachable(); } while (0); } while(0); __clear_bit_unlock(bitnum, addr); do { do { __asm__ __volatile__("": : :"memory"); sub_preempt_count(1); } while (0); __asm__ __volatile__("": : :"memory"); do { if (__builtin_constant_p((((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/bit_spinlock.h", .line = 80, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/bit_spinlock.h", .line = 80, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/bit_spinlock.h", .line = 80, }; ______r = !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/bit_spinlock.h", .line = 80, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) preempt_schedule(); } while (0); } while (0); (void)0; } static inline __attribute__((always_inline)) int bit_spin_is_locked(int bitnum, unsigned long *addr) { return (__builtin_constant_p((bitnum)) ? constant_test_bit((bitnum), (addr)) : variable_test_bit((bitnum), (addr))); } struct hlist_bl_head { struct hlist_bl_node *first; }; struct hlist_bl_node { struct hlist_bl_node *next, **pprev; }; static inline __attribute__((always_inline)) void INIT_HLIST_BL_NODE(struct hlist_bl_node *h) { h->next = ((void *)0); h->pprev = ((void *)0); } static inline __attribute__((always_inline)) int hlist_bl_unhashed(const struct hlist_bl_node *h) { return !h->pprev; } static inline __attribute__((always_inline)) struct hlist_bl_node *hlist_bl_first(struct hlist_bl_head *h) { return (struct hlist_bl_node *) ((unsigned long)h->first & ~1UL); } static inline __attribute__((always_inline)) void hlist_bl_set_first(struct hlist_bl_head *h, struct hlist_bl_node *n) { ; ; h->first = (struct hlist_bl_node *)((unsigned long)n | 1UL); } static inline __attribute__((always_inline)) int hlist_bl_empty(const struct hlist_bl_head *h) { return !((unsigned long)h->first & ~1UL); } static inline __attribute__((always_inline)) void hlist_bl_add_head(struct hlist_bl_node *n, struct hlist_bl_head *h) { struct hlist_bl_node *first = hlist_bl_first(h); n->next = first; if (__builtin_constant_p(((first))) ? !!((first)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/list_bl.h", .line = 82, }; ______r = !!((first)); ______f.miss_hit[______r]++; ______r; })) first->pprev = &n->next; n->pprev = &h->first; hlist_bl_set_first(h, n); } static inline __attribute__((always_inline)) void __hlist_bl_del(struct hlist_bl_node *n) { struct hlist_bl_node *next = n->next; struct hlist_bl_node **pprev = n->pprev; ; *pprev = (struct hlist_bl_node *) ((unsigned long)next | ((unsigned long)*pprev & 1UL)); if (__builtin_constant_p(((next))) ? !!((next)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/list_bl.h", .line = 99, }; ______r = !!((next)); ______f.miss_hit[______r]++; ______r; })) next->pprev = pprev; } static inline __attribute__((always_inline)) void hlist_bl_del(struct hlist_bl_node *n) { __hlist_bl_del(n); n->next = ((void *) 0x00100100 + (0x0UL)); n->pprev = ((void *) 0x00200200 + (0x0UL)); } static inline __attribute__((always_inline)) void hlist_bl_del_init(struct hlist_bl_node *n) { if (__builtin_constant_p(((!hlist_bl_unhashed(n)))) ? !!((!hlist_bl_unhashed(n))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/list_bl.h", .line = 112, }; ______r = !!((!hlist_bl_unhashed(n))); ______f.miss_hit[______r]++; ______r; })) { __hlist_bl_del(n); INIT_HLIST_BL_NODE(n); } } static inline __attribute__((always_inline)) void hlist_bl_lock(struct hlist_bl_head *b) { bit_spin_lock(0, (unsigned long *)b); } static inline __attribute__((always_inline)) void hlist_bl_unlock(struct hlist_bl_head *b) { __bit_spin_unlock(0, (unsigned long *)b); } static inline __attribute__((always_inline)) void hlist_bl_set_first_rcu(struct hlist_bl_head *h, struct hlist_bl_node *n) { ; ; ({ if (__builtin_constant_p(((!__builtin_constant_p(((struct hlist_bl_node *)((unsigned long)n | 1UL))) || ((((struct hlist_bl_node *)((unsigned long)n | 1UL))) != ((void *)0))))) ? !!((!__builtin_constant_p(((struct hlist_bl_node *)((unsigned long)n | 1UL))) || ((((struct hlist_bl_node *)((unsigned long)n | 1UL))) != ((void *)0)))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/rculist_bl.h" , .line = 17 , }; ______r = !!((!__builtin_constant_p(((struct hlist_bl_node *)((unsigned long)n | 1UL))) || ((((struct hlist_bl_node *)((unsigned long)n | 1UL))) != ((void *)0)))); ______f.miss_hit[______r]++; ______r; })) __asm__ __volatile__("": : :"memory"); ((h->first)) = (typeof(*((struct hlist_bl_node *)((unsigned long)n | 1UL))) *)(((struct hlist_bl_node *)((unsigned long)n | 1UL))); }) ; } static inline __attribute__((always_inline)) struct hlist_bl_node *hlist_bl_first_rcu(struct hlist_bl_head *h) { return (struct hlist_bl_node *) ((unsigned long)({ typeof(*(h->first)) *_________p1 = (typeof(*(h->first))* )(*(volatile typeof((h->first)) *)&((h->first))); do { } while (0); ; do { } while (0); ((typeof(*(h->first)) *)(_________p1)); }) & ~1UL); } static inline __attribute__((always_inline)) void hlist_bl_del_init_rcu(struct hlist_bl_node *n) { if (__builtin_constant_p(((!hlist_bl_unhashed(n)))) ? !!((!hlist_bl_unhashed(n))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/rculist_bl.h", .line = 48, }; ______r = !!((!hlist_bl_unhashed(n))); ______f.miss_hit[______r]++; ______r; })) { __hlist_bl_del(n); n->pprev = ((void *)0); } } static inline __attribute__((always_inline)) void hlist_bl_del_rcu(struct hlist_bl_node *n) { __hlist_bl_del(n); n->pprev = ((void *) 0x00200200 + (0x0UL)); } static inline __attribute__((always_inline)) void hlist_bl_add_head_rcu(struct hlist_bl_node *n, struct hlist_bl_head *h) { struct hlist_bl_node *first; first = hlist_bl_first(h); n->next = first; if (__builtin_constant_p(((first))) ? !!((first)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/rculist_bl.h", .line = 107, }; ______r = !!((first)); ______f.miss_hit[______r]++; ______r; })) first->pprev = &n->next; n->pprev = &h->first; hlist_bl_set_first_rcu(h, n); } struct nameidata; struct path; struct vfsmount; struct qstr { unsigned int hash; unsigned int len; const unsigned char *name; }; struct dentry_stat_t { int nr_dentry; int nr_unused; int age_limit; int want_pages; int dummy[2]; }; extern struct dentry_stat_t dentry_stat; static inline __attribute__((always_inline)) int dentry_cmp(const unsigned char *cs, size_t scount, const unsigned char *ct, size_t tcount) { int ret; if (__builtin_constant_p(((scount != tcount))) ? !!((scount != tcount)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/dcache.h", .line = 58, }; ______r = !!((scount != tcount)); ______f.miss_hit[______r]++; ______r; })) return 1; do { ret = (*cs != *ct); if (__builtin_constant_p(((ret))) ? !!((ret)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/dcache.h", .line = 62, }; ______r = !!((ret)); ______f.miss_hit[______r]++; ______r; })) break; cs++; ct++; tcount--; } while (tcount); return ret; } static inline __attribute__((always_inline)) unsigned long partial_name_hash(unsigned long c, unsigned long prevhash) { return (prevhash + (c << 4) + (c >> 4)) * 11; } static inline __attribute__((always_inline)) unsigned long end_name_hash(unsigned long hash) { return (unsigned int) hash; } static inline __attribute__((always_inline)) unsigned int full_name_hash(const unsigned char *name, unsigned int len) { unsigned long hash = 0; while (len--) hash = partial_name_hash(*name++, hash); return end_name_hash(hash); } struct dentry { unsigned int d_flags; seqcount_t d_seq; struct hlist_bl_node d_hash; struct dentry *d_parent; struct qstr d_name; struct inode *d_inode; unsigned char d_iname[36]; unsigned int d_count; spinlock_t d_lock; const struct dentry_operations *d_op; struct super_block *d_sb; unsigned long d_time; void *d_fsdata; struct list_head d_lru; union { struct list_head d_child; struct rcu_head d_rcu; } d_u; struct list_head d_subdirs; struct list_head d_alias; }; enum dentry_d_lock_class { DENTRY_D_LOCK_NORMAL, DENTRY_D_LOCK_NESTED }; struct dentry_operations { int (*d_revalidate)(struct dentry *, struct nameidata *); int (*d_hash)(const struct dentry *, const struct inode *, struct qstr *); int (*d_compare)(const struct dentry *, const struct inode *, const struct dentry *, const struct inode *, unsigned int, const char *, const struct qstr *); int (*d_delete)(const struct dentry *); void (*d_release)(struct dentry *); void (*d_iput)(struct dentry *, struct inode *); char *(*d_dname)(struct dentry *, char *, int); struct vfsmount *(*d_automount)(struct path *); int (*d_manage)(struct dentry *, bool); } __attribute__((__aligned__((1 << (6))))); extern seqlock_t rename_lock; static inline __attribute__((always_inline)) int dname_external(struct dentry *dentry) { return dentry->d_name.name != dentry->d_iname; } extern void d_instantiate(struct dentry *, struct inode *); extern struct dentry * d_instantiate_unique(struct dentry *, struct inode *); extern struct dentry * d_materialise_unique(struct dentry *, struct inode *); extern void __d_drop(struct dentry *dentry); extern void d_drop(struct dentry *dentry); extern void d_delete(struct dentry *); extern void d_set_d_op(struct dentry *dentry, const struct dentry_operations *op); extern struct dentry * d_alloc(struct dentry *, const struct qstr *); extern struct dentry * d_alloc_pseudo(struct super_block *, const struct qstr *); extern struct dentry * d_splice_alias(struct inode *, struct dentry *); extern struct dentry * d_add_ci(struct dentry *, struct inode *, struct qstr *); extern struct dentry * d_obtain_alias(struct inode *); extern void shrink_dcache_sb(struct super_block *); extern void shrink_dcache_parent(struct dentry *); extern void shrink_dcache_for_umount(struct super_block *); extern int d_invalidate(struct dentry *); extern struct dentry * d_alloc_root(struct inode *); extern void d_genocide(struct dentry *); extern struct dentry *d_find_alias(struct inode *); extern void d_prune_aliases(struct inode *); extern int have_submounts(struct dentry *); extern void d_rehash(struct dentry *); static inline __attribute__((always_inline)) void d_add(struct dentry *entry, struct inode *inode) { d_instantiate(entry, inode); d_rehash(entry); } static inline __attribute__((always_inline)) struct dentry *d_add_unique(struct dentry *entry, struct inode *inode) { struct dentry *res; res = d_instantiate_unique(entry, inode); d_rehash(res != ((void *)0) ? res : entry); return res; } extern void dentry_update_name_case(struct dentry *, struct qstr *); extern void d_move(struct dentry *, struct dentry *); extern struct dentry *d_ancestor(struct dentry *, struct dentry *); extern struct dentry *d_lookup(struct dentry *, struct qstr *); extern struct dentry *d_hash_and_lookup(struct dentry *, struct qstr *); extern struct dentry *__d_lookup(struct dentry *, struct qstr *); extern struct dentry *__d_lookup_rcu(struct dentry *parent, struct qstr *name, unsigned *seq, struct inode **inode); static inline __attribute__((always_inline)) int __d_rcu_to_refcount(struct dentry *dentry, unsigned seq) { int ret = 0; assert_spin_locked(&dentry->d_lock); if (__builtin_constant_p(((!read_seqcount_retry(&dentry->d_seq, seq)))) ? !!((!read_seqcount_retry(&dentry->d_seq, seq))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/dcache.h", .line = 327, }; ______r = !!((!read_seqcount_retry(&dentry->d_seq, seq))); ______f.miss_hit[______r]++; ______r; })) { ret = 1; dentry->d_count++; } return ret; } extern int d_validate(struct dentry *, struct dentry *); extern char *dynamic_dname(struct dentry *, char *, int, const char *, ...); extern char *__d_path(const struct path *path, struct path *root, char *, int); extern char *d_path(const struct path *, char *, int); extern char *d_path_with_unreachable(const struct path *, char *, int); extern char *dentry_path_raw(struct dentry *, char *, int); extern char *dentry_path(struct dentry *, char *, int); static inline __attribute__((always_inline)) struct dentry *dget_dlock(struct dentry *dentry) { if (__builtin_constant_p(((dentry))) ? !!((dentry)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/dcache.h", .line = 361, }; ______r = !!((dentry)); ______f.miss_hit[______r]++; ______r; })) dentry->d_count++; return dentry; } static inline __attribute__((always_inline)) struct dentry *dget(struct dentry *dentry) { if (__builtin_constant_p(((dentry))) ? !!((dentry)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/dcache.h", .line = 368, }; ______r = !!((dentry)); ______f.miss_hit[______r]++; ______r; })) { spin_lock(&dentry->d_lock); dget_dlock(dentry); spin_unlock(&dentry->d_lock); } return dentry; } extern struct dentry *dget_parent(struct dentry *dentry); static inline __attribute__((always_inline)) int d_unhashed(struct dentry *dentry) { return hlist_bl_unhashed(&dentry->d_hash); } static inline __attribute__((always_inline)) int d_unlinked(struct dentry *dentry) { return d_unhashed(dentry) && !((dentry) == (dentry)->d_parent); } static inline __attribute__((always_inline)) int cant_mount(struct dentry *dentry) { return (dentry->d_flags & 0x0100); } static inline __attribute__((always_inline)) void dont_mount(struct dentry *dentry) { spin_lock(&dentry->d_lock); dentry->d_flags |= 0x0100; spin_unlock(&dentry->d_lock); } extern void dput(struct dentry *); static inline __attribute__((always_inline)) bool d_managed(struct dentry *dentry) { return dentry->d_flags & (0x10000|0x20000|0x40000); } static inline __attribute__((always_inline)) bool d_mountpoint(struct dentry *dentry) { return dentry->d_flags & 0x10000; } extern struct dentry *lookup_create(struct nameidata *nd, int is_dir); extern int sysctl_vfs_cache_pressure; struct dentry; struct vfsmount; struct path { struct vfsmount *mnt; struct dentry *dentry; }; extern void path_get(struct path *); extern void path_put(struct path *); static inline __attribute__((always_inline)) int path_equal(const struct path *path1, const struct path *path2) { return path1->mnt == path2->mnt && path1->dentry == path2->dentry; } static inline __attribute__((always_inline)) int radix_tree_is_indirect_ptr(void *ptr) { return (int)((unsigned long)ptr & 1); } struct radix_tree_root { unsigned int height; gfp_t gfp_mask; struct radix_tree_node *rnode; }; static inline __attribute__((always_inline)) void *radix_tree_deref_slot(void **pslot) { return ({ typeof(*(*pslot)) *_________p1 = (typeof(*(*pslot))* )(*(volatile typeof((*pslot)) *)&((*pslot))); do { } while (0); ; do { } while (0); ((typeof(*(*pslot)) *)(_________p1)); }); } static inline __attribute__((always_inline)) void *radix_tree_deref_slot_protected(void **pslot, spinlock_t *treelock) { return ({ do { } while (0); ; ((typeof(*(*pslot)) *)((*pslot))); }); } static inline __attribute__((always_inline)) int radix_tree_deref_retry(void *arg) { return (__builtin_constant_p((unsigned long)arg & 1) ? !!((unsigned long)arg & 1) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/radix-tree.h", .line = 173, }; ______r = __builtin_expect(!!((unsigned long)arg & 1), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })); } static inline __attribute__((always_inline)) void radix_tree_replace_slot(void **pslot, void *item) { do { if (__builtin_constant_p((((__builtin_constant_p(radix_tree_is_indirect_ptr(item)) ? !!(radix_tree_is_indirect_ptr(item)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/radix-tree.h", .line = 186, }; ______r = __builtin_expect(!!(radix_tree_is_indirect_ptr(item)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(radix_tree_is_indirect_ptr(item)) ? !!(radix_tree_is_indirect_ptr(item)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/radix-tree.h", .line = 186, }; ______r = __builtin_expect(!!(radix_tree_is_indirect_ptr(item)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/radix-tree.h", .line = 186, }; ______r = !!(((__builtin_constant_p(radix_tree_is_indirect_ptr(item)) ? !!(radix_tree_is_indirect_ptr(item)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/radix-tree.h", .line = 186, }; ______r = __builtin_expect(!!(radix_tree_is_indirect_ptr(item)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) do { asm volatile("1:\tud2\n" ".pushsection __bug_table,\"a\"\n" "2:\t.long 1b, %c0\n" "\t.word %c1, 0\n" "\t.org 2b+%c2\n" ".popsection" : : "i" ("include/linux/radix-tree.h"), "i" (186), "i" (sizeof(struct bug_entry))); __builtin_unreachable(); } while (0); } while(0); ({ if (__builtin_constant_p(((!__builtin_constant_p((item)) || (((item)) != ((void *)0))))) ? !!((!__builtin_constant_p((item)) || (((item)) != ((void *)0)))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/radix-tree.h", .line = 187, }; ______r = !!((!__builtin_constant_p((item)) || (((item)) != ((void *)0)))); ______f.miss_hit[______r]++; ______r; })) __asm__ __volatile__("": : :"memory"); ((*pslot)) = (typeof(*(item)) *)((item)); }); } int radix_tree_insert(struct radix_tree_root *, unsigned long, void *); void *radix_tree_lookup(struct radix_tree_root *, unsigned long); void **radix_tree_lookup_slot(struct radix_tree_root *, unsigned long); void *radix_tree_delete(struct radix_tree_root *, unsigned long); unsigned int radix_tree_gang_lookup(struct radix_tree_root *root, void **results, unsigned long first_index, unsigned int max_items); unsigned int radix_tree_gang_lookup_slot(struct radix_tree_root *root, void ***results, unsigned long first_index, unsigned int max_items); unsigned long radix_tree_next_hole(struct radix_tree_root *root, unsigned long index, unsigned long max_scan); unsigned long radix_tree_prev_hole(struct radix_tree_root *root, unsigned long index, unsigned long max_scan); int radix_tree_preload(gfp_t gfp_mask); void radix_tree_init(void); void *radix_tree_tag_set(struct radix_tree_root *root, unsigned long index, unsigned int tag); void *radix_tree_tag_clear(struct radix_tree_root *root, unsigned long index, unsigned int tag); int radix_tree_tag_get(struct radix_tree_root *root, unsigned long index, unsigned int tag); unsigned int radix_tree_gang_lookup_tag(struct radix_tree_root *root, void **results, unsigned long first_index, unsigned int max_items, unsigned int tag); unsigned int radix_tree_gang_lookup_tag_slot(struct radix_tree_root *root, void ***results, unsigned long first_index, unsigned int max_items, unsigned int tag); unsigned long radix_tree_range_tag_if_tagged(struct radix_tree_root *root, unsigned long *first_indexp, unsigned long last_index, unsigned long nr_to_tag, unsigned int fromtag, unsigned int totag); int radix_tree_tagged(struct radix_tree_root *root, unsigned int tag); static inline __attribute__((always_inline)) void radix_tree_preload_end(void) { do { do { __asm__ __volatile__("": : :"memory"); sub_preempt_count(1); } while (0); __asm__ __volatile__("": : :"memory"); do { if (__builtin_constant_p((((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/radix-tree.h", .line = 228, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/radix-tree.h", .line = 228, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/radix-tree.h", .line = 228, }; ______r = !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/radix-tree.h", .line = 228, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) preempt_schedule(); } while (0); } while (0); } struct semaphore { spinlock_t lock; unsigned int count; struct list_head wait_list; }; static inline __attribute__((always_inline)) void sema_init(struct semaphore *sem, int val) { static struct lock_class_key __key; *sem = (struct semaphore) { .lock = (spinlock_t ) { { .rlock = { .raw_lock = { 0 }, .magic = 0xdead4ead, .owner_cpu = -1, .owner = ((void *)-1L), .dep_map = { .name = "(*sem).lock" } } } }, .count = val, .wait_list = { &((*sem).wait_list), &((*sem).wait_list) }, }; lockdep_init_map(&sem->lock.dep_map, "semaphore->lock", &__key, 0); } extern void down(struct semaphore *sem); extern int __attribute__((warn_unused_result)) down_interruptible(struct semaphore *sem); extern int __attribute__((warn_unused_result)) down_killable(struct semaphore *sem); extern int __attribute__((warn_unused_result)) down_trylock(struct semaphore *sem); extern int __attribute__((warn_unused_result)) down_timeout(struct semaphore *sem, long jiffies); extern void up(struct semaphore *sem); struct fiemap_extent { __u64 fe_logical; __u64 fe_physical; __u64 fe_length; __u64 fe_reserved64[2]; __u32 fe_flags; __u32 fe_reserved[3]; }; struct fiemap { __u64 fm_start; __u64 fm_length; __u32 fm_flags; __u32 fm_mapped_extents; __u32 fm_extent_count; __u32 fm_reserved; struct fiemap_extent fm_extents[0]; }; struct export_operations; struct hd_geometry; struct iovec; struct nameidata; struct kiocb; struct kobject; struct pipe_inode_info; struct poll_table_struct; struct kstatfs; struct vm_area_struct; struct vfsmount; struct cred; extern void __attribute__ ((__section__(".init.text"))) __attribute__((__cold__)) __attribute__((no_instrument_function)) inode_init(void); extern void __attribute__ ((__section__(".init.text"))) __attribute__((__cold__)) __attribute__((no_instrument_function)) inode_init_early(void); extern void __attribute__ ((__section__(".init.text"))) __attribute__((__cold__)) __attribute__((no_instrument_function)) files_init(unsigned long); extern struct files_stat_struct files_stat; extern unsigned long get_max_files(void); extern int sysctl_nr_open; extern struct inodes_stat_t inodes_stat; extern int leases_enable, lease_break_time; struct buffer_head; typedef int (get_block_t)(struct inode *inode, sector_t iblock, struct buffer_head *bh_result, int create); typedef void (dio_iodone_t)(struct kiocb *iocb, loff_t offset, ssize_t bytes, void *private, int ret, bool is_async); struct iattr { unsigned int ia_valid; umode_t ia_mode; uid_t ia_uid; gid_t ia_gid; loff_t ia_size; struct timespec ia_atime; struct timespec ia_mtime; struct timespec ia_ctime; struct file *ia_file; }; enum { QIF_BLIMITS_B = 0, QIF_SPACE_B, QIF_ILIMITS_B, QIF_INODES_B, QIF_BTIME_B, QIF_ITIME_B, }; struct if_dqblk { __u64 dqb_bhardlimit; __u64 dqb_bsoftlimit; __u64 dqb_curspace; __u64 dqb_ihardlimit; __u64 dqb_isoftlimit; __u64 dqb_curinodes; __u64 dqb_btime; __u64 dqb_itime; __u32 dqb_valid; }; struct if_dqinfo { __u64 dqi_bgrace; __u64 dqi_igrace; __u32 dqi_flags; __u32 dqi_valid; }; enum { QUOTA_NL_C_UNSPEC, QUOTA_NL_C_WARNING, __QUOTA_NL_C_MAX, }; enum { QUOTA_NL_A_UNSPEC, QUOTA_NL_A_QTYPE, QUOTA_NL_A_EXCESS_ID, QUOTA_NL_A_WARNING, QUOTA_NL_A_DEV_MAJOR, QUOTA_NL_A_DEV_MINOR, QUOTA_NL_A_CAUSED_ID, __QUOTA_NL_A_MAX, }; typedef struct fs_disk_quota { __s8 d_version; __s8 d_flags; __u16 d_fieldmask; __u32 d_id; __u64 d_blk_hardlimit; __u64 d_blk_softlimit; __u64 d_ino_hardlimit; __u64 d_ino_softlimit; __u64 d_bcount; __u64 d_icount; __s32 d_itimer; __s32 d_btimer; __u16 d_iwarns; __u16 d_bwarns; __s32 d_padding2; __u64 d_rtb_hardlimit; __u64 d_rtb_softlimit; __u64 d_rtbcount; __s32 d_rtbtimer; __u16 d_rtbwarns; __s16 d_padding3; char d_padding4[8]; } fs_disk_quota_t; typedef struct fs_qfilestat { __u64 qfs_ino; __u64 qfs_nblks; __u32 qfs_nextents; } fs_qfilestat_t; typedef struct fs_quota_stat { __s8 qs_version; __u16 qs_flags; __s8 qs_pad; fs_qfilestat_t qs_uquota; fs_qfilestat_t qs_gquota; __u32 qs_incoredqs; __s32 qs_btimelimit; __s32 qs_itimelimit; __s32 qs_rtbtimelimit; __u16 qs_bwarnlimit; __u16 qs_iwarnlimit; } fs_quota_stat_t; struct dquot; struct qtree_fmt_operations { void (*mem2disk_dqblk)(void *disk, struct dquot *dquot); void (*disk2mem_dqblk)(struct dquot *dquot, void *disk); int (*is_id)(void *disk, struct dquot *dquot); }; struct qtree_mem_dqinfo { struct super_block *dqi_sb; int dqi_type; unsigned int dqi_blocks; unsigned int dqi_free_blk; unsigned int dqi_free_entry; unsigned int dqi_blocksize_bits; unsigned int dqi_entry_size; unsigned int dqi_usable_bs; unsigned int dqi_qtree_depth; struct qtree_fmt_operations *dqi_ops; }; int qtree_write_dquot(struct qtree_mem_dqinfo *info, struct dquot *dquot); int qtree_read_dquot(struct qtree_mem_dqinfo *info, struct dquot *dquot); int qtree_delete_dquot(struct qtree_mem_dqinfo *info, struct dquot *dquot); int qtree_release_dquot(struct qtree_mem_dqinfo *info, struct dquot *dquot); int qtree_entry_unused(struct qtree_mem_dqinfo *info, char *disk); static inline __attribute__((always_inline)) int qtree_depth(struct qtree_mem_dqinfo *info) { unsigned int epb = info->dqi_usable_bs >> 2; unsigned long long entries = epb; int i; for (i = 1; entries < (1ULL << 32); i++) entries *= epb; return i; } typedef __kernel_uid32_t qid_t; typedef long long qsize_t; extern spinlock_t dq_data_lock; struct mem_dqblk { qsize_t dqb_bhardlimit; qsize_t dqb_bsoftlimit; qsize_t dqb_curspace; qsize_t dqb_rsvspace; qsize_t dqb_ihardlimit; qsize_t dqb_isoftlimit; qsize_t dqb_curinodes; time_t dqb_btime; time_t dqb_itime; }; struct quota_format_type; struct mem_dqinfo { struct quota_format_type *dqi_format; int dqi_fmt_id; struct list_head dqi_dirty_list; unsigned long dqi_flags; unsigned int dqi_bgrace; unsigned int dqi_igrace; qsize_t dqi_maxblimit; qsize_t dqi_maxilimit; void *dqi_priv; }; struct super_block; extern void mark_info_dirty(struct super_block *sb, int type); static inline __attribute__((always_inline)) int info_dirty(struct mem_dqinfo *info) { return (__builtin_constant_p((16)) ? constant_test_bit((16), (&info->dqi_flags)) : variable_test_bit((16), (&info->dqi_flags))); } enum { DQST_LOOKUPS, DQST_DROPS, DQST_READS, DQST_WRITES, DQST_CACHE_HITS, DQST_ALLOC_DQUOTS, DQST_FREE_DQUOTS, DQST_SYNCS, _DQST_DQSTAT_LAST }; struct dqstats { int stat[_DQST_DQSTAT_LAST]; struct percpu_counter counter[_DQST_DQSTAT_LAST]; }; extern struct dqstats *dqstats_pcpu; extern struct dqstats dqstats; static inline __attribute__((always_inline)) void dqstats_inc(unsigned int type) { percpu_counter_inc(&dqstats.counter[type]); } static inline __attribute__((always_inline)) void dqstats_dec(unsigned int type) { percpu_counter_dec(&dqstats.counter[type]); } struct dquot { struct hlist_node dq_hash; struct list_head dq_inuse; struct list_head dq_free; struct list_head dq_dirty; struct mutex dq_lock; atomic_t dq_count; wait_queue_head_t dq_wait_unused; struct super_block *dq_sb; unsigned int dq_id; loff_t dq_off; unsigned long dq_flags; short dq_type; struct mem_dqblk dq_dqb; }; struct quota_format_ops { int (*check_quota_file)(struct super_block *sb, int type); int (*read_file_info)(struct super_block *sb, int type); int (*write_file_info)(struct super_block *sb, int type); int (*free_file_info)(struct super_block *sb, int type); int (*read_dqblk)(struct dquot *dquot); int (*commit_dqblk)(struct dquot *dquot); int (*release_dqblk)(struct dquot *dquot); }; struct dquot_operations { int (*write_dquot) (struct dquot *); struct dquot *(*alloc_dquot)(struct super_block *, int); void (*destroy_dquot)(struct dquot *); int (*acquire_dquot) (struct dquot *); int (*release_dquot) (struct dquot *); int (*mark_dirty) (struct dquot *); int (*write_info) (struct super_block *, int); qsize_t *(*get_reserved_space) (struct inode *); }; struct path; struct quotactl_ops { int (*quota_on)(struct super_block *, int, int, struct path *); int (*quota_on_meta)(struct super_block *, int, int); int (*quota_off)(struct super_block *, int); int (*quota_sync)(struct super_block *, int, int); int (*get_info)(struct super_block *, int, struct if_dqinfo *); int (*set_info)(struct super_block *, int, struct if_dqinfo *); int (*get_dqblk)(struct super_block *, int, qid_t, struct fs_disk_quota *); int (*set_dqblk)(struct super_block *, int, qid_t, struct fs_disk_quota *); int (*get_xstate)(struct super_block *, struct fs_quota_stat *); int (*set_xstate)(struct super_block *, unsigned int, int); }; struct quota_format_type { int qf_fmt_id; const struct quota_format_ops *qf_ops; struct module *qf_owner; struct quota_format_type *qf_next; }; enum { _DQUOT_USAGE_ENABLED = 0, _DQUOT_LIMITS_ENABLED, _DQUOT_SUSPENDED, _DQUOT_STATE_FLAGS }; static inline __attribute__((always_inline)) unsigned int dquot_state_flag(unsigned int flags, int type) { return flags << _DQUOT_STATE_FLAGS * type; } static inline __attribute__((always_inline)) unsigned int dquot_generic_flag(unsigned int flags, int type) { return (flags >> _DQUOT_STATE_FLAGS * type) & ((1 << _DQUOT_USAGE_ENABLED) | (1 << _DQUOT_LIMITS_ENABLED) | (1 << _DQUOT_SUSPENDED)); } static inline __attribute__((always_inline)) void quota_send_warning(short type, unsigned int id, dev_t dev, const char warntype) { return; } struct quota_info { unsigned int flags; struct mutex dqio_mutex; struct mutex dqonoff_mutex; struct rw_semaphore dqptr_sem; struct inode *files[2]; struct mem_dqinfo info[2]; const struct quota_format_ops *ops[2]; }; int register_quota_format(struct quota_format_type *fmt); void unregister_quota_format(struct quota_format_type *fmt); struct quota_module_name { int qm_fmt_id; char *qm_mod_name; }; enum positive_aop_returns { AOP_WRITEPAGE_ACTIVATE = 0x80000, AOP_TRUNCATED_PAGE = 0x80001, }; struct page; struct address_space; struct writeback_control; struct iov_iter { const struct iovec *iov; unsigned long nr_segs; size_t iov_offset; size_t count; }; size_t iov_iter_copy_from_user_atomic(struct page *page, struct iov_iter *i, unsigned long offset, size_t bytes); size_t iov_iter_copy_from_user(struct page *page, struct iov_iter *i, unsigned long offset, size_t bytes); void iov_iter_advance(struct iov_iter *i, size_t bytes); int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes); size_t iov_iter_single_seg_count(struct iov_iter *i); static inline __attribute__((always_inline)) void iov_iter_init(struct iov_iter *i, const struct iovec *iov, unsigned long nr_segs, size_t count, size_t written) { i->iov = iov; i->nr_segs = nr_segs; i->iov_offset = 0; i->count = count + written; iov_iter_advance(i, written); } static inline __attribute__((always_inline)) size_t iov_iter_count(struct iov_iter *i) { return i->count; } typedef struct { size_t written; size_t count; union { char *buf; void *data; } arg; int error; } read_descriptor_t; typedef int (*read_actor_t)(read_descriptor_t *, struct page *, unsigned long, unsigned long); struct address_space_operations { int (*writepage)(struct page *page, struct writeback_control *wbc); int (*readpage)(struct file *, struct page *); int (*writepages)(struct address_space *, struct writeback_control *); int (*set_page_dirty)(struct page *page); int (*readpages)(struct file *filp, struct address_space *mapping, struct list_head *pages, unsigned nr_pages); int (*write_begin)(struct file *, struct address_space *mapping, loff_t pos, unsigned len, unsigned flags, struct page **pagep, void **fsdata); int (*write_end)(struct file *, struct address_space *mapping, loff_t pos, unsigned len, unsigned copied, struct page *page, void *fsdata); sector_t (*bmap)(struct address_space *, sector_t); void (*invalidatepage) (struct page *, unsigned long); int (*releasepage) (struct page *, gfp_t); void (*freepage)(struct page *); ssize_t (*direct_IO)(int, struct kiocb *, const struct iovec *iov, loff_t offset, unsigned long nr_segs); int (*get_xip_mem)(struct address_space *, unsigned long, int, void **, unsigned long *); int (*migratepage) (struct address_space *, struct page *, struct page *); int (*launder_page) (struct page *); int (*is_partially_uptodate) (struct page *, read_descriptor_t *, unsigned long); int (*error_remove_page)(struct address_space *, struct page *); }; extern const struct address_space_operations empty_aops; int pagecache_write_begin(struct file *, struct address_space *mapping, loff_t pos, unsigned len, unsigned flags, struct page **pagep, void **fsdata); int pagecache_write_end(struct file *, struct address_space *mapping, loff_t pos, unsigned len, unsigned copied, struct page *page, void *fsdata); struct backing_dev_info; struct address_space { struct inode *host; struct radix_tree_root page_tree; spinlock_t tree_lock; unsigned int i_mmap_writable; struct prio_tree_root i_mmap; struct list_head i_mmap_nonlinear; struct mutex i_mmap_mutex; unsigned long nrpages; unsigned long writeback_index; const struct address_space_operations *a_ops; unsigned long flags; struct backing_dev_info *backing_dev_info; spinlock_t private_lock; struct list_head private_list; struct address_space *assoc_mapping; } __attribute__((aligned(sizeof(long)))); struct block_device { dev_t bd_dev; int bd_openers; struct inode * bd_inode; struct super_block * bd_super; struct mutex bd_mutex; struct list_head bd_inodes; void * bd_claiming; void * bd_holder; int bd_holders; bool bd_write_holder; struct list_head bd_holder_disks; struct block_device * bd_contains; unsigned bd_block_size; struct hd_struct * bd_part; unsigned bd_part_count; int bd_invalidated; struct gendisk * bd_disk; struct list_head bd_list; unsigned long bd_private; int bd_fsfreeze_count; struct mutex bd_fsfreeze_mutex; }; int mapping_tagged(struct address_space *mapping, int tag); static inline __attribute__((always_inline)) int mapping_mapped(struct address_space *mapping) { return !prio_tree_empty(&mapping->i_mmap) || !list_empty(&mapping->i_mmap_nonlinear); } static inline __attribute__((always_inline)) int mapping_writably_mapped(struct address_space *mapping) { return mapping->i_mmap_writable != 0; } struct posix_acl; struct inode { umode_t i_mode; uid_t i_uid; gid_t i_gid; const struct inode_operations *i_op; struct super_block *i_sb; spinlock_t i_lock; unsigned int i_flags; unsigned long i_state; void *i_security; struct mutex i_mutex; unsigned long dirtied_when; struct hlist_node i_hash; struct list_head i_wb_list; struct list_head i_lru; struct list_head i_sb_list; union { struct list_head i_dentry; struct rcu_head i_rcu; }; unsigned long i_ino; atomic_t i_count; unsigned int i_nlink; dev_t i_rdev; unsigned int i_blkbits; u64 i_version; loff_t i_size; seqcount_t i_size_seqcount; struct timespec i_atime; struct timespec i_mtime; struct timespec i_ctime; blkcnt_t i_blocks; unsigned short i_bytes; struct rw_semaphore i_alloc_sem; const struct file_operations *i_fop; struct file_lock *i_flock; struct address_space *i_mapping; struct address_space i_data; struct dquot *i_dquot[2]; struct list_head i_devices; union { struct pipe_inode_info *i_pipe; struct block_device *i_bdev; struct cdev *i_cdev; }; __u32 i_generation; __u32 i_fsnotify_mask; struct hlist_head i_fsnotify_marks; atomic_t i_writecount; struct posix_acl *i_acl; struct posix_acl *i_default_acl; void *i_private; }; static inline __attribute__((always_inline)) int inode_unhashed(struct inode *inode) { return hlist_unhashed(&inode->i_hash); } enum inode_i_mutex_lock_class { I_MUTEX_NORMAL, I_MUTEX_PARENT, I_MUTEX_CHILD, I_MUTEX_XATTR, I_MUTEX_QUOTA }; static inline __attribute__((always_inline)) loff_t i_size_read(const struct inode *inode) { loff_t i_size; unsigned int seq; do { seq = read_seqcount_begin(&inode->i_size_seqcount); i_size = inode->i_size; } while (read_seqcount_retry(&inode->i_size_seqcount, seq)); return i_size; } static inline __attribute__((always_inline)) void i_size_write(struct inode *inode, loff_t i_size) { write_seqcount_begin(&inode->i_size_seqcount); inode->i_size = i_size; write_seqcount_end(&inode->i_size_seqcount); } static inline __attribute__((always_inline)) unsigned iminor(const struct inode *inode) { return ((unsigned int) ((inode->i_rdev) & ((1U << 20) - 1))); } static inline __attribute__((always_inline)) unsigned imajor(const struct inode *inode) { return ((unsigned int) ((inode->i_rdev) >> 20)); } extern struct block_device *I_BDEV(struct inode *inode); struct fown_struct { rwlock_t lock; struct pid *pid; enum pid_type pid_type; uid_t uid, euid; int signum; }; struct file_ra_state { unsigned long start; unsigned int size; unsigned int async_size; unsigned int ra_pages; unsigned int mmap_miss; loff_t prev_pos; }; static inline __attribute__((always_inline)) int ra_has_index(struct file_ra_state *ra, unsigned long index) { return (index >= ra->start && index < ra->start + ra->size); } struct file { union { struct list_head fu_list; struct rcu_head fu_rcuhead; } f_u; struct path f_path; const struct file_operations *f_op; spinlock_t f_lock; int f_sb_list_cpu; atomic_long_t f_count; unsigned int f_flags; fmode_t f_mode; loff_t f_pos; struct fown_struct f_owner; const struct cred *f_cred; struct file_ra_state f_ra; u64 f_version; void *f_security; void *private_data; struct list_head f_ep_links; struct address_space *f_mapping; }; struct file_handle { __u32 handle_bytes; int handle_type; unsigned char f_handle[0]; }; static inline __attribute__((always_inline)) void file_take_write(struct file *filp) {} static inline __attribute__((always_inline)) void file_release_write(struct file *filp) {} static inline __attribute__((always_inline)) void file_reset_write(struct file *filp) {} static inline __attribute__((always_inline)) void file_check_state(struct file *filp) {} static inline __attribute__((always_inline)) int file_check_writeable(struct file *filp) { return 0; } typedef struct files_struct *fl_owner_t; struct file_lock_operations { void (*fl_copy_lock)(struct file_lock *, struct file_lock *); void (*fl_release_private)(struct file_lock *); }; struct lock_manager_operations { int (*fl_compare_owner)(struct file_lock *, struct file_lock *); void (*fl_notify)(struct file_lock *); int (*fl_grant)(struct file_lock *, struct file_lock *, int); void (*fl_release_private)(struct file_lock *); void (*fl_break)(struct file_lock *); int (*fl_change)(struct file_lock **, int); }; struct lock_manager { struct list_head list; }; void locks_start_grace(struct lock_manager *); void locks_end_grace(struct lock_manager *); int locks_in_grace(void); enum nfs_stat { NFS_OK = 0, NFSERR_PERM = 1, NFSERR_NOENT = 2, NFSERR_IO = 5, NFSERR_NXIO = 6, NFSERR_EAGAIN = 11, NFSERR_ACCES = 13, NFSERR_EXIST = 17, NFSERR_XDEV = 18, NFSERR_NODEV = 19, NFSERR_NOTDIR = 20, NFSERR_ISDIR = 21, NFSERR_INVAL = 22, NFSERR_FBIG = 27, NFSERR_NOSPC = 28, NFSERR_ROFS = 30, NFSERR_MLINK = 31, NFSERR_OPNOTSUPP = 45, NFSERR_NAMETOOLONG = 63, NFSERR_NOTEMPTY = 66, NFSERR_DQUOT = 69, NFSERR_STALE = 70, NFSERR_REMOTE = 71, NFSERR_WFLUSH = 99, NFSERR_BADHANDLE = 10001, NFSERR_NOT_SYNC = 10002, NFSERR_BAD_COOKIE = 10003, NFSERR_NOTSUPP = 10004, NFSERR_TOOSMALL = 10005, NFSERR_SERVERFAULT = 10006, NFSERR_BADTYPE = 10007, NFSERR_JUKEBOX = 10008, NFSERR_SAME = 10009, NFSERR_DENIED = 10010, NFSERR_EXPIRED = 10011, NFSERR_LOCKED = 10012, NFSERR_GRACE = 10013, NFSERR_FHEXPIRED = 10014, NFSERR_SHARE_DENIED = 10015, NFSERR_WRONGSEC = 10016, NFSERR_CLID_INUSE = 10017, NFSERR_RESOURCE = 10018, NFSERR_MOVED = 10019, NFSERR_NOFILEHANDLE = 10020, NFSERR_MINOR_VERS_MISMATCH = 10021, NFSERR_STALE_CLIENTID = 10022, NFSERR_STALE_STATEID = 10023, NFSERR_OLD_STATEID = 10024, NFSERR_BAD_STATEID = 10025, NFSERR_BAD_SEQID = 10026, NFSERR_NOT_SAME = 10027, NFSERR_LOCK_RANGE = 10028, NFSERR_SYMLINK = 10029, NFSERR_RESTOREFH = 10030, NFSERR_LEASE_MOVED = 10031, NFSERR_ATTRNOTSUPP = 10032, NFSERR_NO_GRACE = 10033, NFSERR_RECLAIM_BAD = 10034, NFSERR_RECLAIM_CONFLICT = 10035, NFSERR_BAD_XDR = 10036, NFSERR_LOCKS_HELD = 10037, NFSERR_OPENMODE = 10038, NFSERR_BADOWNER = 10039, NFSERR_BADCHAR = 10040, NFSERR_BADNAME = 10041, NFSERR_BAD_RANGE = 10042, NFSERR_LOCK_NOTSUPP = 10043, NFSERR_OP_ILLEGAL = 10044, NFSERR_DEADLOCK = 10045, NFSERR_FILE_OPEN = 10046, NFSERR_ADMIN_REVOKED = 10047, NFSERR_CB_PATH_DOWN = 10048, }; enum nfs_ftype { NFNON = 0, NFREG = 1, NFDIR = 2, NFBLK = 3, NFCHR = 4, NFLNK = 5, NFSOCK = 6, NFBAD = 7, NFFIFO = 8 }; typedef u32 rpc_authflavor_t; enum rpc_auth_flavors { RPC_AUTH_NULL = 0, RPC_AUTH_UNIX = 1, RPC_AUTH_SHORT = 2, RPC_AUTH_DES = 3, RPC_AUTH_KRB = 4, RPC_AUTH_GSS = 6, RPC_AUTH_MAXFLAVOR = 8, RPC_AUTH_GSS_KRB5 = 390003, RPC_AUTH_GSS_KRB5I = 390004, RPC_AUTH_GSS_KRB5P = 390005, RPC_AUTH_GSS_LKEY = 390006, RPC_AUTH_GSS_LKEYI = 390007, RPC_AUTH_GSS_LKEYP = 390008, RPC_AUTH_GSS_SPKM = 390009, RPC_AUTH_GSS_SPKMI = 390010, RPC_AUTH_GSS_SPKMP = 390011, }; enum rpc_msg_type { RPC_CALL = 0, RPC_REPLY = 1 }; enum rpc_reply_stat { RPC_MSG_ACCEPTED = 0, RPC_MSG_DENIED = 1 }; enum rpc_accept_stat { RPC_SUCCESS = 0, RPC_PROG_UNAVAIL = 1, RPC_PROG_MISMATCH = 2, RPC_PROC_UNAVAIL = 3, RPC_GARBAGE_ARGS = 4, RPC_SYSTEM_ERR = 5, RPC_DROP_REPLY = 60000, }; enum rpc_reject_stat { RPC_MISMATCH = 0, RPC_AUTH_ERROR = 1 }; enum rpc_auth_stat { RPC_AUTH_OK = 0, RPC_AUTH_BADCRED = 1, RPC_AUTH_REJECTEDCRED = 2, RPC_AUTH_BADVERF = 3, RPC_AUTH_REJECTEDVERF = 4, RPC_AUTH_TOOWEAK = 5, RPCSEC_GSS_CREDPROBLEM = 13, RPCSEC_GSS_CTXPROBLEM = 14 }; typedef __be32 rpc_fraghdr; extern __be32 in_aton(const char *str); extern int in4_pton(const char *src, int srclen, u8 *dst, int delim, const char **end); extern int in6_pton(const char *src, int srclen, u8 *dst, int delim, const char **end); struct nfs_fh { unsigned short size; unsigned char data[128]; }; static inline __attribute__((always_inline)) int nfs_compare_fh(const struct nfs_fh *a, const struct nfs_fh *b) { return a->size != b->size || __builtin_memcmp(a->data, b->data, a->size) != 0; } static inline __attribute__((always_inline)) void nfs_copy_fh(struct nfs_fh *target, const struct nfs_fh *source) { target->size = source->size; __builtin_memcpy(target->data, source->data, source->size); } enum nfs3_stable_how { NFS_UNSTABLE = 0, NFS_DATA_SYNC = 1, NFS_FILE_SYNC = 2 }; struct nlm_lockowner; struct nfs_lock_info { u32 state; struct nlm_lockowner *owner; struct list_head list; }; struct nfs4_lock_state; struct nfs4_lock_info { struct nfs4_lock_state *owner; }; struct file_lock { struct file_lock *fl_next; struct list_head fl_link; struct list_head fl_block; fl_owner_t fl_owner; unsigned char fl_flags; unsigned char fl_type; unsigned int fl_pid; struct pid *fl_nspid; wait_queue_head_t fl_wait; struct file *fl_file; loff_t fl_start; loff_t fl_end; struct fasync_struct * fl_fasync; unsigned long fl_break_time; const struct file_lock_operations *fl_ops; const struct lock_manager_operations *fl_lmops; union { struct nfs_lock_info nfs_fl; struct nfs4_lock_info nfs4_fl; struct { struct list_head link; int state; } afs; } fl_u; }; struct f_owner_ex { int type; __kernel_pid_t pid; }; struct flock { short l_type; short l_whence; __kernel_off_t l_start; __kernel_off_t l_len; __kernel_pid_t l_pid; }; struct flock64 { short l_type; short l_whence; __kernel_loff_t l_start; __kernel_loff_t l_len; __kernel_pid_t l_pid; }; extern void send_sigio(struct fown_struct *fown, int fd, int band); extern int fcntl_getlk(struct file *, struct flock *); extern int fcntl_setlk(unsigned int, struct file *, unsigned int, struct flock *); extern int fcntl_getlk64(struct file *, struct flock64 *); extern int fcntl_setlk64(unsigned int, struct file *, unsigned int, struct flock64 *); extern int fcntl_setlease(unsigned int fd, struct file *filp, long arg); extern int fcntl_getlease(struct file *filp); void locks_free_lock(struct file_lock *fl); extern void locks_init_lock(struct file_lock *); extern struct file_lock * locks_alloc_lock(void); extern void locks_copy_lock(struct file_lock *, struct file_lock *); extern void __locks_copy_lock(struct file_lock *, const struct file_lock *); extern void locks_remove_posix(struct file *, fl_owner_t); extern void locks_remove_flock(struct file *); extern void locks_release_private(struct file_lock *); extern void posix_test_lock(struct file *, struct file_lock *); extern int posix_lock_file(struct file *, struct file_lock *, struct file_lock *); extern int posix_lock_file_wait(struct file *, struct file_lock *); extern int posix_unblock_lock(struct file *, struct file_lock *); extern int vfs_test_lock(struct file *, struct file_lock *); extern int vfs_lock_file(struct file *, unsigned int, struct file_lock *, struct file_lock *); extern int vfs_cancel_lock(struct file *filp, struct file_lock *fl); extern int flock_lock_file_wait(struct file *filp, struct file_lock *fl); extern int __break_lease(struct inode *inode, unsigned int flags); extern void lease_get_mtime(struct inode *, struct timespec *time); extern int generic_setlease(struct file *, long, struct file_lock **); extern int vfs_setlease(struct file *, long, struct file_lock **); extern int lease_modify(struct file_lock **, int); extern int lock_may_read(struct inode *, loff_t start, unsigned long count); extern int lock_may_write(struct inode *, loff_t start, unsigned long count); extern void lock_flocks(void); extern void unlock_flocks(void); struct fasync_struct { spinlock_t fa_lock; int magic; int fa_fd; struct fasync_struct *fa_next; struct file *fa_file; struct rcu_head fa_rcu; }; extern int fasync_helper(int, struct file *, int, struct fasync_struct **); extern struct fasync_struct *fasync_insert_entry(int, struct file *, struct fasync_struct **, struct fasync_struct *); extern int fasync_remove_entry(struct file *, struct fasync_struct **); extern struct fasync_struct *fasync_alloc(void); extern void fasync_free(struct fasync_struct *); extern void kill_fasync(struct fasync_struct **, int, int); extern int __f_setown(struct file *filp, struct pid *, enum pid_type, int force); extern int f_setown(struct file *filp, unsigned long arg, int force); extern void f_delown(struct file *filp); extern pid_t f_getown(struct file *filp); extern int send_sigurg(struct fown_struct *fown); extern struct list_head super_blocks; extern spinlock_t sb_lock; struct super_block { struct list_head s_list; dev_t s_dev; unsigned char s_dirt; unsigned char s_blocksize_bits; unsigned long s_blocksize; loff_t s_maxbytes; struct file_system_type *s_type; const struct super_operations *s_op; const struct dquot_operations *dq_op; const struct quotactl_ops *s_qcop; const struct export_operations *s_export_op; unsigned long s_flags; unsigned long s_magic; struct dentry *s_root; struct rw_semaphore s_umount; struct mutex s_lock; int s_count; atomic_t s_active; void *s_security; const struct xattr_handler **s_xattr; struct list_head s_inodes; struct hlist_bl_head s_anon; struct list_head *s_files; struct list_head s_dentry_lru; int s_nr_dentry_unused; struct block_device *s_bdev; struct backing_dev_info *s_bdi; struct mtd_info *s_mtd; struct list_head s_instances; struct quota_info s_dquot; int s_frozen; wait_queue_head_t s_wait_unfrozen; char s_id[32]; u8 s_uuid[16]; void *s_fs_info; fmode_t s_mode; u32 s_time_gran; struct mutex s_vfs_rename_mutex; char *s_subtype; char *s_options; const struct dentry_operations *s_d_op; int cleancache_poolid; }; extern struct timespec current_fs_time(struct super_block *sb); enum { SB_UNFROZEN = 0, SB_FREEZE_WRITE = 1, SB_FREEZE_TRANS = 2, }; extern struct user_namespace init_user_ns; extern bool inode_owner_or_capable(const struct inode *inode); extern void lock_super(struct super_block *); extern void unlock_super(struct super_block *); extern int vfs_create(struct inode *, struct dentry *, int, struct nameidata *); extern int vfs_mkdir(struct inode *, struct dentry *, int); extern int vfs_mknod(struct inode *, struct dentry *, int, dev_t); extern int vfs_symlink(struct inode *, struct dentry *, const char *); extern int vfs_link(struct dentry *, struct inode *, struct dentry *); extern int vfs_rmdir(struct inode *, struct dentry *); extern int vfs_unlink(struct inode *, struct dentry *); extern int vfs_rename(struct inode *, struct dentry *, struct inode *, struct dentry *); extern void dentry_unhash(struct dentry *dentry); extern int file_permission(struct file *, int); extern void inode_init_owner(struct inode *inode, const struct inode *dir, mode_t mode); struct fiemap_extent_info { unsigned int fi_flags; unsigned int fi_extents_mapped; unsigned int fi_extents_max; struct fiemap_extent *fi_extents_start; }; int fiemap_fill_next_extent(struct fiemap_extent_info *info, u64 logical, u64 phys, u64 len, u32 flags); int fiemap_check_flags(struct fiemap_extent_info *fieinfo, u32 fs_flags); typedef int (*filldir_t)(void *, const char *, int, loff_t, u64, unsigned); struct block_device_operations; struct file_operations { struct module *owner; loff_t (*llseek) (struct file *, loff_t, int); ssize_t (*read) (struct file *, char *, size_t, loff_t *); ssize_t (*write) (struct file *, const char *, size_t, loff_t *); ssize_t (*aio_read) (struct kiocb *, const struct iovec *, unsigned long, loff_t); ssize_t (*aio_write) (struct kiocb *, const struct iovec *, unsigned long, loff_t); int (*readdir) (struct file *, void *, filldir_t); unsigned int (*poll) (struct file *, struct poll_table_struct *); long (*unlocked_ioctl) (struct file *, unsigned int, unsigned long); long (*compat_ioctl) (struct file *, unsigned int, unsigned long); int (*mmap) (struct file *, struct vm_area_struct *); int (*open) (struct inode *, struct file *); int (*flush) (struct file *, fl_owner_t id); int (*release) (struct inode *, struct file *); int (*fsync) (struct file *, int datasync); int (*aio_fsync) (struct kiocb *, int datasync); int (*fasync) (int, struct file *, int); int (*lock) (struct file *, int, struct file_lock *); ssize_t (*sendpage) (struct file *, struct page *, int, size_t, loff_t *, int); unsigned long (*get_unmapped_area)(struct file *, unsigned long, unsigned long, unsigned long, unsigned long); int (*check_flags)(int); int (*flock) (struct file *, int, struct file_lock *); ssize_t (*splice_write)(struct pipe_inode_info *, struct file *, loff_t *, size_t, unsigned int); ssize_t (*splice_read)(struct file *, loff_t *, struct pipe_inode_info *, size_t, unsigned int); int (*setlease)(struct file *, long, struct file_lock **); long (*fallocate)(struct file *file, int mode, loff_t offset, loff_t len); }; struct inode_operations { struct dentry * (*lookup) (struct inode *,struct dentry *, struct nameidata *); void * (*follow_link) (struct dentry *, struct nameidata *); int (*permission) (struct inode *, int, unsigned int); int (*check_acl)(struct inode *, int, unsigned int); int (*readlink) (struct dentry *, char *,int); void (*put_link) (struct dentry *, struct nameidata *, void *); int (*create) (struct inode *,struct dentry *,int, struct nameidata *); int (*link) (struct dentry *,struct inode *,struct dentry *); int (*unlink) (struct inode *,struct dentry *); int (*symlink) (struct inode *,struct dentry *,const char *); int (*mkdir) (struct inode *,struct dentry *,int); int (*rmdir) (struct inode *,struct dentry *); int (*mknod) (struct inode *,struct dentry *,int,dev_t); int (*rename) (struct inode *, struct dentry *, struct inode *, struct dentry *); void (*truncate) (struct inode *); int (*setattr) (struct dentry *, struct iattr *); int (*getattr) (struct vfsmount *mnt, struct dentry *, struct kstat *); int (*setxattr) (struct dentry *, const char *,const void *,size_t,int); ssize_t (*getxattr) (struct dentry *, const char *, void *, size_t); ssize_t (*listxattr) (struct dentry *, char *, size_t); int (*removexattr) (struct dentry *, const char *); void (*truncate_range)(struct inode *, loff_t, loff_t); int (*fiemap)(struct inode *, struct fiemap_extent_info *, u64 start, u64 len); } __attribute__((__aligned__((1 << (6))))); struct seq_file; ssize_t rw_copy_check_uvector(int type, const struct iovec * uvector, unsigned long nr_segs, unsigned long fast_segs, struct iovec *fast_pointer, struct iovec **ret_pointer); extern ssize_t vfs_read(struct file *, char *, size_t, loff_t *); extern ssize_t vfs_write(struct file *, const char *, size_t, loff_t *); extern ssize_t vfs_readv(struct file *, const struct iovec *, unsigned long, loff_t *); extern ssize_t vfs_writev(struct file *, const struct iovec *, unsigned long, loff_t *); struct super_operations { struct inode *(*alloc_inode)(struct super_block *sb); void (*destroy_inode)(struct inode *); void (*dirty_inode) (struct inode *, int flags); int (*write_inode) (struct inode *, struct writeback_control *wbc); int (*drop_inode) (struct inode *); void (*evict_inode) (struct inode *); void (*put_super) (struct super_block *); void (*write_super) (struct super_block *); int (*sync_fs)(struct super_block *sb, int wait); int (*freeze_fs) (struct super_block *); int (*unfreeze_fs) (struct super_block *); int (*statfs) (struct dentry *, struct kstatfs *); int (*remount_fs) (struct super_block *, int *, char *); void (*umount_begin) (struct super_block *); int (*show_options)(struct seq_file *, struct vfsmount *); int (*show_devname)(struct seq_file *, struct vfsmount *); int (*show_path)(struct seq_file *, struct vfsmount *); int (*show_stats)(struct seq_file *, struct vfsmount *); ssize_t (*quota_read)(struct super_block *, int, char *, size_t, loff_t); ssize_t (*quota_write)(struct super_block *, int, const char *, size_t, loff_t); int (*bdev_try_to_free_page)(struct super_block*, struct page*, gfp_t); }; extern void __mark_inode_dirty(struct inode *, int); static inline __attribute__((always_inline)) void mark_inode_dirty(struct inode *inode) { __mark_inode_dirty(inode, ((1 << 0) | (1 << 1) | (1 << 2))); } static inline __attribute__((always_inline)) void mark_inode_dirty_sync(struct inode *inode) { __mark_inode_dirty(inode, (1 << 0)); } static inline __attribute__((always_inline)) void inc_nlink(struct inode *inode) { inode->i_nlink++; } static inline __attribute__((always_inline)) void inode_inc_link_count(struct inode *inode) { inc_nlink(inode); mark_inode_dirty(inode); } static inline __attribute__((always_inline)) void drop_nlink(struct inode *inode) { inode->i_nlink--; } static inline __attribute__((always_inline)) void clear_nlink(struct inode *inode) { inode->i_nlink = 0; } static inline __attribute__((always_inline)) void inode_dec_link_count(struct inode *inode) { drop_nlink(inode); mark_inode_dirty(inode); } static inline __attribute__((always_inline)) void inode_inc_iversion(struct inode *inode) { spin_lock(&inode->i_lock); inode->i_version++; spin_unlock(&inode->i_lock); } extern void touch_atime(struct vfsmount *mnt, struct dentry *dentry); static inline __attribute__((always_inline)) void file_accessed(struct file *file) { if (__builtin_constant_p(((!(file->f_flags & 01000000)))) ? !!((!(file->f_flags & 01000000))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/fs.h", .line = 1795, }; ______r = !!((!(file->f_flags & 01000000))); ______f.miss_hit[______r]++; ______r; })) touch_atime(file->f_path.mnt, file->f_path.dentry); } int sync_inode(struct inode *inode, struct writeback_control *wbc); int sync_inode_metadata(struct inode *inode, int wait); struct file_system_type { const char *name; int fs_flags; struct dentry *(*mount) (struct file_system_type *, int, const char *, void *); void (*kill_sb) (struct super_block *); struct module *owner; struct file_system_type * next; struct list_head fs_supers; struct lock_class_key s_lock_key; struct lock_class_key s_umount_key; struct lock_class_key s_vfs_rename_key; struct lock_class_key i_lock_key; struct lock_class_key i_mutex_key; struct lock_class_key i_mutex_dir_key; struct lock_class_key i_alloc_sem_key; }; extern struct dentry *mount_ns(struct file_system_type *fs_type, int flags, void *data, int (*fill_super)(struct super_block *, void *, int)); extern struct dentry *mount_bdev(struct file_system_type *fs_type, int flags, const char *dev_name, void *data, int (*fill_super)(struct super_block *, void *, int)); extern struct dentry *mount_single(struct file_system_type *fs_type, int flags, void *data, int (*fill_super)(struct super_block *, void *, int)); extern struct dentry *mount_nodev(struct file_system_type *fs_type, int flags, void *data, int (*fill_super)(struct super_block *, void *, int)); void generic_shutdown_super(struct super_block *sb); void kill_block_super(struct super_block *sb); void kill_anon_super(struct super_block *sb); void kill_litter_super(struct super_block *sb); void deactivate_super(struct super_block *sb); void deactivate_locked_super(struct super_block *sb); int set_anon_super(struct super_block *s, void *data); struct super_block *sget(struct file_system_type *type, int (*test)(struct super_block *,void *), int (*set)(struct super_block *,void *), void *data); extern struct dentry *mount_pseudo(struct file_system_type *, char *, const struct super_operations *ops, const struct dentry_operations *dops, unsigned long); static inline __attribute__((always_inline)) void sb_mark_dirty(struct super_block *sb) { sb->s_dirt = 1; } static inline __attribute__((always_inline)) void sb_mark_clean(struct super_block *sb) { sb->s_dirt = 0; } static inline __attribute__((always_inline)) int sb_is_dirty(struct super_block *sb) { return sb->s_dirt; } extern int register_filesystem(struct file_system_type *); extern int unregister_filesystem(struct file_system_type *); extern struct vfsmount *kern_mount_data(struct file_system_type *, void *data); extern int may_umount_tree(struct vfsmount *); extern int may_umount(struct vfsmount *); extern long do_mount(char *, char *, char *, unsigned long, void *); extern struct vfsmount *collect_mounts(struct path *); extern void drop_collected_mounts(struct vfsmount *); extern int iterate_mounts(int (*)(struct vfsmount *, void *), void *, struct vfsmount *); extern int vfs_statfs(struct path *, struct kstatfs *); extern int user_statfs(const char *, struct kstatfs *); extern int fd_statfs(int, struct kstatfs *); extern int statfs_by_dentry(struct dentry *, struct kstatfs *); extern int freeze_super(struct super_block *super); extern int thaw_super(struct super_block *super); extern int current_umask(void); extern struct kobject *fs_kobj; extern int rw_verify_area(int, struct file *, loff_t *, size_t); extern int locks_mandatory_locked(struct inode *); extern int locks_mandatory_area(int, struct inode *, struct file *, loff_t, size_t); static inline __attribute__((always_inline)) int __mandatory_lock(struct inode *ino) { return (ino->i_mode & (0002000 | 00010)) == 0002000; } static inline __attribute__((always_inline)) int mandatory_lock(struct inode *ino) { return ((ino)->i_sb->s_flags & (64)) && __mandatory_lock(ino); } static inline __attribute__((always_inline)) int locks_verify_locked(struct inode *inode) { if (__builtin_constant_p(((mandatory_lock(inode)))) ? !!((mandatory_lock(inode))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/fs.h", .line = 1923, }; ______r = !!((mandatory_lock(inode))); ______f.miss_hit[______r]++; ______r; })) return locks_mandatory_locked(inode); return 0; } static inline __attribute__((always_inline)) int locks_verify_truncate(struct inode *inode, struct file *filp, loff_t size) { if (__builtin_constant_p(((inode->i_flock && mandatory_lock(inode)))) ? !!((inode->i_flock && mandatory_lock(inode))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/fs.h", .line = 1932, }; ______r = !!((inode->i_flock && mandatory_lock(inode))); ______f.miss_hit[______r]++; ______r; })) return locks_mandatory_area( 2, inode, filp, size < inode->i_size ? size : inode->i_size, (size < inode->i_size ? inode->i_size - size : size - inode->i_size) ); return 0; } static inline __attribute__((always_inline)) int break_lease(struct inode *inode, unsigned int mode) { if (__builtin_constant_p(((inode->i_flock))) ? !!((inode->i_flock)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/fs.h", .line = 1944, }; ______r = !!((inode->i_flock)); ______f.miss_hit[______r]++; ______r; })) return __break_lease(inode, mode); return 0; } extern int do_truncate(struct dentry *, loff_t start, unsigned int time_attrs, struct file *filp); extern int do_fallocate(struct file *file, int mode, loff_t offset, loff_t len); extern long do_sys_open(int dfd, const char *filename, int flags, int mode); extern struct file *filp_open(const char *, int, int); extern struct file *file_open_root(struct dentry *, struct vfsmount *, const char *, int); extern struct file * dentry_open(struct dentry *, struct vfsmount *, int, const struct cred *); extern int filp_close(struct file *, fl_owner_t id); extern char * getname(const char *); extern int ioctl_preallocate(struct file *filp, void *argp); extern void __attribute__ ((__section__(".init.text"))) __attribute__((__cold__)) __attribute__((no_instrument_function)) vfs_caches_init_early(void); extern void __attribute__ ((__section__(".init.text"))) __attribute__((__cold__)) __attribute__((no_instrument_function)) vfs_caches_init(unsigned long); extern struct kmem_cache *names_cachep; extern void putname(const char *name); extern int register_blkdev(unsigned int, const char *); extern void unregister_blkdev(unsigned int, const char *); extern struct block_device *bdget(dev_t); extern struct block_device *bdgrab(struct block_device *bdev); extern void bd_set_size(struct block_device *, loff_t size); extern void bd_forget(struct inode *inode); extern void bdput(struct block_device *); extern void invalidate_bdev(struct block_device *); extern int sync_blockdev(struct block_device *bdev); extern struct super_block *freeze_bdev(struct block_device *); extern void emergency_thaw_all(void); extern int thaw_bdev(struct block_device *bdev, struct super_block *sb); extern int fsync_bdev(struct block_device *); extern int sync_filesystem(struct super_block *); extern const struct file_operations def_blk_fops; extern const struct file_operations def_chr_fops; extern const struct file_operations bad_sock_fops; extern const struct file_operations def_fifo_fops; extern int ioctl_by_bdev(struct block_device *, unsigned, unsigned long); extern int blkdev_ioctl(struct block_device *, fmode_t, unsigned, unsigned long); extern long compat_blkdev_ioctl(struct file *, unsigned, unsigned long); extern int blkdev_get(struct block_device *bdev, fmode_t mode, void *holder); extern struct block_device *blkdev_get_by_path(const char *path, fmode_t mode, void *holder); extern struct block_device *blkdev_get_by_dev(dev_t dev, fmode_t mode, void *holder); extern int blkdev_put(struct block_device *bdev, fmode_t mode); extern int bd_link_disk_holder(struct block_device *bdev, struct gendisk *disk); extern void bd_unlink_disk_holder(struct block_device *bdev, struct gendisk *disk); extern int alloc_chrdev_region(dev_t *, unsigned, unsigned, const char *); extern int register_chrdev_region(dev_t, unsigned, const char *); extern int __register_chrdev(unsigned int major, unsigned int baseminor, unsigned int count, const char *name, const struct file_operations *fops); extern void __unregister_chrdev(unsigned int major, unsigned int baseminor, unsigned int count, const char *name); extern void unregister_chrdev_region(dev_t, unsigned); extern void chrdev_show(struct seq_file *,off_t); static inline __attribute__((always_inline)) int register_chrdev(unsigned int major, const char *name, const struct file_operations *fops) { return __register_chrdev(major, 0, 256, name, fops); } static inline __attribute__((always_inline)) void unregister_chrdev(unsigned int major, const char *name) { __unregister_chrdev(major, 0, 256, name); } extern const char *__bdevname(dev_t, char *buffer); extern const char *bdevname(struct block_device *bdev, char *buffer); extern struct block_device *lookup_bdev(const char *); extern void blkdev_show(struct seq_file *,off_t); extern void init_special_inode(struct inode *, umode_t, dev_t); extern void make_bad_inode(struct inode *); extern int is_bad_inode(struct inode *); extern const struct file_operations read_pipefifo_fops; extern const struct file_operations write_pipefifo_fops; extern const struct file_operations rdwr_pipefifo_fops; extern int fs_may_remount_ro(struct super_block *); extern void check_disk_size_change(struct gendisk *disk, struct block_device *bdev); extern int revalidate_disk(struct gendisk *); extern int check_disk_change(struct block_device *); extern int __invalidate_device(struct block_device *, bool); extern int invalidate_partition(struct gendisk *, int); unsigned long invalidate_mapping_pages(struct address_space *mapping, unsigned long start, unsigned long end); static inline __attribute__((always_inline)) void invalidate_remote_inode(struct inode *inode) { if (__builtin_constant_p((((((inode->i_mode) & 00170000) == 0100000) || (((inode->i_mode) & 00170000) == 0040000) || (((inode->i_mode) & 00170000) == 0120000)))) ? !!(((((inode->i_mode) & 00170000) == 0100000) || (((inode->i_mode) & 00170000) == 0040000) || (((inode->i_mode) & 00170000) == 0120000))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/fs.h" , .line = 2159 , }; ______r = !!(((((inode->i_mode) & 00170000) == 0100000) || (((inode->i_mode) & 00170000) == 0040000) || (((inode->i_mode) & 00170000) == 0120000))); ______f.miss_hit[______r]++; ______r; })) invalidate_mapping_pages(inode->i_mapping, 0, -1); } extern int invalidate_inode_pages2(struct address_space *mapping); extern int invalidate_inode_pages2_range(struct address_space *mapping, unsigned long start, unsigned long end); extern int write_inode_now(struct inode *, int); extern int filemap_fdatawrite(struct address_space *); extern int filemap_flush(struct address_space *); extern int filemap_fdatawait(struct address_space *); extern int filemap_fdatawait_range(struct address_space *, loff_t lstart, loff_t lend); extern int filemap_write_and_wait(struct address_space *mapping); extern int filemap_write_and_wait_range(struct address_space *mapping, loff_t lstart, loff_t lend); extern int __filemap_fdatawrite_range(struct address_space *mapping, loff_t start, loff_t end, int sync_mode); extern int filemap_fdatawrite_range(struct address_space *mapping, loff_t start, loff_t end); extern int vfs_fsync_range(struct file *file, loff_t start, loff_t end, int datasync); extern int vfs_fsync(struct file *file, int datasync); extern int generic_write_sync(struct file *file, loff_t pos, loff_t count); extern void sync_supers(void); extern void emergency_sync(void); extern void emergency_remount(void); extern sector_t bmap(struct inode *, sector_t); extern int notify_change(struct dentry *, struct iattr *); extern int inode_permission(struct inode *, int); extern int generic_permission(struct inode *, int, unsigned int, int (*check_acl)(struct inode *, int, unsigned int)); static inline __attribute__((always_inline)) bool execute_ok(struct inode *inode) { return (inode->i_mode & (00100|00010|00001)) || (((inode->i_mode) & 00170000) == 0040000); } extern int get_write_access(struct inode *); extern int deny_write_access(struct file *); static inline __attribute__((always_inline)) void put_write_access(struct inode * inode) { atomic_dec(&inode->i_writecount); } static inline __attribute__((always_inline)) void allow_write_access(struct file *file) { if (__builtin_constant_p(((file))) ? !!((file)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/fs.h", .line = 2207, }; ______r = !!((file)); ______f.miss_hit[______r]++; ______r; })) atomic_inc(&file->f_path.dentry->d_inode->i_writecount); } static inline __attribute__((always_inline)) void i_readcount_dec(struct inode *inode) { return; } static inline __attribute__((always_inline)) void i_readcount_inc(struct inode *inode) { return; } extern int do_pipe_flags(int *, int); extern struct file *create_read_pipe(struct file *f, int flags); extern struct file *create_write_pipe(int flags); extern void free_write_pipe(struct file *); extern int kernel_read(struct file *, loff_t, char *, unsigned long); extern struct file * open_exec(const char *); extern int is_subdir(struct dentry *, struct dentry *); extern int path_is_under(struct path *, struct path *); extern ino_t find_inode_number(struct dentry *, struct qstr *); extern loff_t default_llseek(struct file *file, loff_t offset, int origin); extern loff_t vfs_llseek(struct file *file, loff_t offset, int origin); extern int inode_init_always(struct super_block *, struct inode *); extern void inode_init_once(struct inode *); extern void address_space_init_once(struct address_space *mapping); extern void ihold(struct inode * inode); extern void iput(struct inode *); extern struct inode * igrab(struct inode *); extern ino_t iunique(struct super_block *, ino_t); extern int inode_needs_sync(struct inode *inode); extern int generic_delete_inode(struct inode *inode); extern int generic_drop_inode(struct inode *inode); extern struct inode *ilookup5_nowait(struct super_block *sb, unsigned long hashval, int (*test)(struct inode *, void *), void *data); extern struct inode *ilookup5(struct super_block *sb, unsigned long hashval, int (*test)(struct inode *, void *), void *data); extern struct inode *ilookup(struct super_block *sb, unsigned long ino); extern struct inode * iget5_locked(struct super_block *, unsigned long, int (*test)(struct inode *, void *), int (*set)(struct inode *, void *), void *); extern struct inode * iget_locked(struct super_block *, unsigned long); extern int insert_inode_locked4(struct inode *, unsigned long, int (*test)(struct inode *, void *), void *); extern int insert_inode_locked(struct inode *); extern void unlock_new_inode(struct inode *); extern unsigned int get_next_ino(void); extern void __iget(struct inode * inode); extern void iget_failed(struct inode *); extern void end_writeback(struct inode *); extern void __destroy_inode(struct inode *); extern struct inode *new_inode(struct super_block *); extern void free_inode_nonrcu(struct inode *inode); extern int should_remove_suid(struct dentry *); extern int file_remove_suid(struct file *); extern void __insert_inode_hash(struct inode *, unsigned long hashval); extern void remove_inode_hash(struct inode *); static inline __attribute__((always_inline)) void insert_inode_hash(struct inode *inode) { __insert_inode_hash(inode, inode->i_ino); } extern void inode_sb_list_add(struct inode *inode); extern void submit_bio(int, struct bio *); extern int bdev_read_only(struct block_device *); extern int set_blocksize(struct block_device *, int); extern int sb_set_blocksize(struct super_block *, int); extern int sb_min_blocksize(struct super_block *, int); extern int generic_file_mmap(struct file *, struct vm_area_struct *); extern int generic_file_readonly_mmap(struct file *, struct vm_area_struct *); extern int file_read_actor(read_descriptor_t * desc, struct page *page, unsigned long offset, unsigned long size); int generic_write_checks(struct file *file, loff_t *pos, size_t *count, int isblk); extern ssize_t generic_file_aio_read(struct kiocb *, const struct iovec *, unsigned long, loff_t); extern ssize_t __generic_file_aio_write(struct kiocb *, const struct iovec *, unsigned long, loff_t *); extern ssize_t generic_file_aio_write(struct kiocb *, const struct iovec *, unsigned long, loff_t); extern ssize_t generic_file_direct_write(struct kiocb *, const struct iovec *, unsigned long *, loff_t, loff_t *, size_t, size_t); extern ssize_t generic_file_buffered_write(struct kiocb *, const struct iovec *, unsigned long, loff_t, loff_t *, size_t, ssize_t); extern ssize_t do_sync_read(struct file *filp, char *buf, size_t len, loff_t *ppos); extern ssize_t do_sync_write(struct file *filp, const char *buf, size_t len, loff_t *ppos); extern int generic_segment_checks(const struct iovec *iov, unsigned long *nr_segs, size_t *count, int access_flags); extern ssize_t blkdev_aio_write(struct kiocb *iocb, const struct iovec *iov, unsigned long nr_segs, loff_t pos); extern int blkdev_fsync(struct file *filp, int datasync); extern ssize_t generic_file_splice_read(struct file *, loff_t *, struct pipe_inode_info *, size_t, unsigned int); extern ssize_t default_file_splice_read(struct file *, loff_t *, struct pipe_inode_info *, size_t, unsigned int); extern ssize_t generic_file_splice_write(struct pipe_inode_info *, struct file *, loff_t *, size_t, unsigned int); extern ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out, loff_t *, size_t len, unsigned int flags); extern long do_splice_direct(struct file *in, loff_t *ppos, struct file *out, size_t len, unsigned int flags); extern void file_ra_state_init(struct file_ra_state *ra, struct address_space *mapping); extern loff_t noop_llseek(struct file *file, loff_t offset, int origin); extern loff_t no_llseek(struct file *file, loff_t offset, int origin); extern loff_t generic_file_llseek(struct file *file, loff_t offset, int origin); extern loff_t generic_file_llseek_unlocked(struct file *file, loff_t offset, int origin); extern int generic_file_open(struct inode * inode, struct file * filp); extern int nonseekable_open(struct inode * inode, struct file * filp); static inline __attribute__((always_inline)) int xip_truncate_page(struct address_space *mapping, loff_t from) { return 0; } typedef void (dio_submit_t)(int rw, struct bio *bio, struct inode *inode, loff_t file_offset); enum { DIO_LOCKING = 0x01, DIO_SKIP_HOLES = 0x02, }; void dio_end_io(struct bio *bio, int error); ssize_t __blockdev_direct_IO(int rw, struct kiocb *iocb, struct inode *inode, struct block_device *bdev, const struct iovec *iov, loff_t offset, unsigned long nr_segs, get_block_t get_block, dio_iodone_t end_io, dio_submit_t submit_io, int flags); static inline __attribute__((always_inline)) ssize_t blockdev_direct_IO(int rw, struct kiocb *iocb, struct inode *inode, struct block_device *bdev, const struct iovec *iov, loff_t offset, unsigned long nr_segs, get_block_t get_block, dio_iodone_t end_io) { return __blockdev_direct_IO(rw, iocb, inode, bdev, iov, offset, nr_segs, get_block, end_io, ((void *)0), DIO_LOCKING | DIO_SKIP_HOLES); } extern const struct file_operations generic_ro_fops; extern int vfs_readlink(struct dentry *, char *, int, const char *); extern int vfs_follow_link(struct nameidata *, const char *); extern int page_readlink(struct dentry *, char *, int); extern void *page_follow_link_light(struct dentry *, struct nameidata *); extern void page_put_link(struct dentry *, struct nameidata *, void *); extern int __page_symlink(struct inode *inode, const char *symname, int len, int nofs); extern int page_symlink(struct inode *inode, const char *symname, int len); extern const struct inode_operations page_symlink_inode_operations; extern int generic_readlink(struct dentry *, char *, int); extern void generic_fillattr(struct inode *, struct kstat *); extern int vfs_getattr(struct vfsmount *, struct dentry *, struct kstat *); void __inode_add_bytes(struct inode *inode, loff_t bytes); void inode_add_bytes(struct inode *inode, loff_t bytes); void inode_sub_bytes(struct inode *inode, loff_t bytes); loff_t inode_get_bytes(struct inode *inode); void inode_set_bytes(struct inode *inode, loff_t bytes); extern int vfs_readdir(struct file *, filldir_t, void *); extern int vfs_stat(const char *, struct kstat *); extern int vfs_lstat(const char *, struct kstat *); extern int vfs_fstat(unsigned int, struct kstat *); extern int vfs_fstatat(int , const char *, struct kstat *, int); extern int do_vfs_ioctl(struct file *filp, unsigned int fd, unsigned int cmd, unsigned long arg); extern int __generic_block_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, loff_t start, loff_t len, get_block_t *get_block); extern int generic_block_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo, u64 start, u64 len, get_block_t *get_block); extern void get_filesystem(struct file_system_type *fs); extern void put_filesystem(struct file_system_type *fs); extern struct file_system_type *get_fs_type(const char *name); extern struct super_block *get_super(struct block_device *); extern struct super_block *get_active_super(struct block_device *bdev); extern struct super_block *user_get_super(dev_t); extern void drop_super(struct super_block *sb); extern void iterate_supers(void (*)(struct super_block *, void *), void *); extern int dcache_dir_open(struct inode *, struct file *); extern int dcache_dir_close(struct inode *, struct file *); extern loff_t dcache_dir_lseek(struct file *, loff_t, int); extern int dcache_readdir(struct file *, void *, filldir_t); extern int simple_setattr(struct dentry *, struct iattr *); extern int simple_getattr(struct vfsmount *, struct dentry *, struct kstat *); extern int simple_statfs(struct dentry *, struct kstatfs *); extern int simple_link(struct dentry *, struct inode *, struct dentry *); extern int simple_unlink(struct inode *, struct dentry *); extern int simple_rmdir(struct inode *, struct dentry *); extern int simple_rename(struct inode *, struct dentry *, struct inode *, struct dentry *); extern int noop_fsync(struct file *, int); extern int simple_empty(struct dentry *); extern int simple_readpage(struct file *file, struct page *page); extern int simple_write_begin(struct file *file, struct address_space *mapping, loff_t pos, unsigned len, unsigned flags, struct page **pagep, void **fsdata); extern int simple_write_end(struct file *file, struct address_space *mapping, loff_t pos, unsigned len, unsigned copied, struct page *page, void *fsdata); extern struct dentry *simple_lookup(struct inode *, struct dentry *, struct nameidata *); extern ssize_t generic_read_dir(struct file *, char *, size_t, loff_t *); extern const struct file_operations simple_dir_operations; extern const struct inode_operations simple_dir_inode_operations; struct tree_descr { char *name; const struct file_operations *ops; int mode; }; struct dentry *d_alloc_name(struct dentry *, const char *); extern int simple_fill_super(struct super_block *, unsigned long, struct tree_descr *); extern int simple_pin_fs(struct file_system_type *, struct vfsmount **mount, int *count); extern void simple_release_fs(struct vfsmount **mount, int *count); extern ssize_t simple_read_from_buffer(void *to, size_t count, loff_t *ppos, const void *from, size_t available); extern ssize_t simple_write_to_buffer(void *to, size_t available, loff_t *ppos, const void *from, size_t count); extern int generic_file_fsync(struct file *, int); extern int generic_check_addressable(unsigned, u64); extern int buffer_migrate_page(struct address_space *, struct page *, struct page *); extern int inode_change_ok(const struct inode *, struct iattr *); extern int inode_newsize_ok(const struct inode *, loff_t offset); extern void setattr_copy(struct inode *inode, const struct iattr *attr); extern void file_update_time(struct file *file); extern int generic_show_options(struct seq_file *m, struct vfsmount *mnt); extern void save_mount_options(struct super_block *sb, char *options); extern void replace_mount_options(struct super_block *sb, char *options); static inline __attribute__((always_inline)) ino_t parent_ino(struct dentry *dentry) { ino_t res; spin_lock(&dentry->d_lock); res = dentry->d_parent->d_inode->i_ino; spin_unlock(&dentry->d_lock); return res; } struct simple_transaction_argresp { ssize_t size; char data[0]; }; char *simple_transaction_get(struct file *file, const char *buf, size_t size); ssize_t simple_transaction_read(struct file *file, char *buf, size_t size, loff_t *pos); int simple_transaction_release(struct inode *inode, struct file *file); void simple_transaction_set(struct file *file, size_t n); static inline __attribute__((always_inline)) void __attribute__((format(printf, 1, 2))) __simple_attr_check_format(const char *fmt, ...) { } int simple_attr_open(struct inode *inode, struct file *file, int (*get)(void *, u64 *), int (*set)(void *, u64), const char *fmt); int simple_attr_release(struct inode *inode, struct file *file); ssize_t simple_attr_read(struct file *file, char *buf, size_t len, loff_t *ppos); ssize_t simple_attr_write(struct file *file, const char *buf, size_t len, loff_t *ppos); struct ctl_table; int proc_nr_files(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos); int proc_nr_dentry(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos); int proc_nr_inodes(struct ctl_table *table, int write, void *buffer, size_t *lenp, loff_t *ppos); int __attribute__ ((__section__(".init.text"))) __attribute__((__cold__)) __attribute__((no_instrument_function)) get_filesystem_list(char *buf); static inline __attribute__((always_inline)) int is_sxid(mode_t mode) { return (mode & 0004000) || ((mode & 0002000) && (mode & 00010)); } static inline __attribute__((always_inline)) void inode_has_no_xattr(struct inode *inode) { if (__builtin_constant_p(((!is_sxid(inode->i_mode) && (inode->i_sb->s_flags & (1<<28))))) ? !!((!is_sxid(inode->i_mode) && (inode->i_sb->s_flags & (1<<28)))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/fs.h", .line = 2597, }; ______r = !!((!is_sxid(inode->i_mode) && (inode->i_sb->s_flags & (1<<28)))); ______f.miss_hit[______r]++; ______r; })) inode->i_flags |= 4096; } struct range { u64 start; u64 end; }; int add_range(struct range *range, int az, int nr_range, u64 start, u64 end); int add_range_with_merge(struct range *range, int az, int nr_range, u64 start, u64 end); void subtract_range(struct range *range, int az, u64 start, u64 end); int clean_sort_range(struct range *range, int az); void sort_range(struct range *range, int nr_range); static inline __attribute__((always_inline)) resource_size_t cap_resource(u64 val) { if (__builtin_constant_p(((val > ((resource_size_t)~0)))) ? !!((val > ((resource_size_t)~0))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/range.h", .line = 25, }; ______r = !!((val > ((resource_size_t)~0))); ______f.miss_hit[______r]++; ______r; })) return ((resource_size_t)~0); return val; } struct mempolicy; struct anon_vma; struct file_ra_state; struct user_struct; struct writeback_control; extern unsigned long max_mapnr; extern unsigned long num_physpages; extern unsigned long totalram_pages; extern void * high_memory; extern int page_cluster; extern int sysctl_legacy_va_layout; extern unsigned long empty_zero_page[((1UL) << 12) / sizeof(unsigned long)]; extern spinlock_t pgd_lock; extern struct list_head pgd_list; extern struct mm_struct *pgd_page_get_mm(struct page *page); static inline __attribute__((always_inline)) int pte_dirty(pte_t pte) { return pte_flags(pte) & (((pteval_t)(1)) << 6); } static inline __attribute__((always_inline)) int pte_young(pte_t pte) { return pte_flags(pte) & (((pteval_t)(1)) << 5); } static inline __attribute__((always_inline)) int pmd_young(pmd_t pmd) { return pmd_flags(pmd) & (((pteval_t)(1)) << 5); } static inline __attribute__((always_inline)) int pte_write(pte_t pte) { return pte_flags(pte) & (((pteval_t)(1)) << 1); } static inline __attribute__((always_inline)) int pte_file(pte_t pte) { return pte_flags(pte) & (((pteval_t)(1)) << 6); } static inline __attribute__((always_inline)) int pte_huge(pte_t pte) { return pte_flags(pte) & (((pteval_t)(1)) << 7); } static inline __attribute__((always_inline)) int pte_global(pte_t pte) { return pte_flags(pte) & (((pteval_t)(1)) << 8); } static inline __attribute__((always_inline)) int pte_exec(pte_t pte) { return !(pte_flags(pte) & (((pteval_t)(1)) << 63)); } static inline __attribute__((always_inline)) int pte_special(pte_t pte) { return pte_flags(pte) & (((pteval_t)(1)) << 9); } static inline __attribute__((always_inline)) unsigned long pte_pfn(pte_t pte) { return (pte_val(pte) & ((pteval_t)(((signed long)(~(((1UL) << 12)-1))) & ((phys_addr_t)((1ULL << 44) - 1))))) >> 12; } static inline __attribute__((always_inline)) unsigned long pmd_pfn(pmd_t pmd) { return (pmd_val(pmd) & ((pteval_t)(((signed long)(~(((1UL) << 12)-1))) & ((phys_addr_t)((1ULL << 44) - 1))))) >> 12; } static inline __attribute__((always_inline)) int pmd_large(pmd_t pte) { return (pmd_flags(pte) & ((((pteval_t)(1)) << 7) | (((pteval_t)(1)) << 0))) == ((((pteval_t)(1)) << 7) | (((pteval_t)(1)) << 0)); } static inline __attribute__((always_inline)) int pmd_trans_splitting(pmd_t pmd) { return pmd_val(pmd) & (((pteval_t)(1)) << 9); } static inline __attribute__((always_inline)) int pmd_trans_huge(pmd_t pmd) { return pmd_val(pmd) & (((pteval_t)(1)) << 7); } static inline __attribute__((always_inline)) int has_transparent_hugepage(void) { return (__builtin_constant_p((0*32+ 3)) && ( ((((0*32+ 3))>>5)==0 && (1UL<<(((0*32+ 3))&31) & ((1<<((0*32+ 0) & 31))|0|0|(1<<((0*32+ 6) & 31))| (1<<((0*32+ 8) & 31))|0|0|(1<<((0*32+15) & 31))| 0|0))) || ((((0*32+ 3))>>5)==1 && (1UL<<(((0*32+ 3))&31) & (0|0))) || ((((0*32+ 3))>>5)==2 && (1UL<<(((0*32+ 3))&31) & 0)) || ((((0*32+ 3))>>5)==3 && (1UL<<(((0*32+ 3))&31) & (0))) || ((((0*32+ 3))>>5)==4 && (1UL<<(((0*32+ 3))&31) & 0)) || ((((0*32+ 3))>>5)==5 && (1UL<<(((0*32+ 3))&31) & 0)) || ((((0*32+ 3))>>5)==6 && (1UL<<(((0*32+ 3))&31) & 0)) || ((((0*32+ 3))>>5)==7 && (1UL<<(((0*32+ 3))&31) & 0)) || ((((0*32+ 3))>>5)==8 && (1UL<<(((0*32+ 3))&31) & 0)) || ((((0*32+ 3))>>5)==9 && (1UL<<(((0*32+ 3))&31) & 0)) ) ? 1 : (__builtin_constant_p(((0*32+ 3))) ? constant_test_bit(((0*32+ 3)), ((unsigned long *)((&boot_cpu_data)->x86_capability))) : variable_test_bit(((0*32+ 3)), ((unsigned long *)((&boot_cpu_data)->x86_capability))))); } static inline __attribute__((always_inline)) pte_t pte_set_flags(pte_t pte, pteval_t set) { pteval_t v = native_pte_val(pte); return native_make_pte(v | set); } static inline __attribute__((always_inline)) pte_t pte_clear_flags(pte_t pte, pteval_t clear) { pteval_t v = native_pte_val(pte); return native_make_pte(v & ~clear); } static inline __attribute__((always_inline)) pte_t pte_mkclean(pte_t pte) { return pte_clear_flags(pte, (((pteval_t)(1)) << 6)); } static inline __attribute__((always_inline)) pte_t pte_mkold(pte_t pte) { return pte_clear_flags(pte, (((pteval_t)(1)) << 5)); } static inline __attribute__((always_inline)) pte_t pte_wrprotect(pte_t pte) { return pte_clear_flags(pte, (((pteval_t)(1)) << 1)); } static inline __attribute__((always_inline)) pte_t pte_mkexec(pte_t pte) { return pte_clear_flags(pte, (((pteval_t)(1)) << 63)); } static inline __attribute__((always_inline)) pte_t pte_mkdirty(pte_t pte) { return pte_set_flags(pte, (((pteval_t)(1)) << 6)); } static inline __attribute__((always_inline)) pte_t pte_mkyoung(pte_t pte) { return pte_set_flags(pte, (((pteval_t)(1)) << 5)); } static inline __attribute__((always_inline)) pte_t pte_mkwrite(pte_t pte) { return pte_set_flags(pte, (((pteval_t)(1)) << 1)); } static inline __attribute__((always_inline)) pte_t pte_mkhuge(pte_t pte) { return pte_set_flags(pte, (((pteval_t)(1)) << 7)); } static inline __attribute__((always_inline)) pte_t pte_clrhuge(pte_t pte) { return pte_clear_flags(pte, (((pteval_t)(1)) << 7)); } static inline __attribute__((always_inline)) pte_t pte_mkglobal(pte_t pte) { return pte_set_flags(pte, (((pteval_t)(1)) << 8)); } static inline __attribute__((always_inline)) pte_t pte_clrglobal(pte_t pte) { return pte_clear_flags(pte, (((pteval_t)(1)) << 8)); } static inline __attribute__((always_inline)) pte_t pte_mkspecial(pte_t pte) { return pte_set_flags(pte, (((pteval_t)(1)) << 9)); } static inline __attribute__((always_inline)) pmd_t pmd_set_flags(pmd_t pmd, pmdval_t set) { pmdval_t v = native_pmd_val(pmd); return __pmd(v | set); } static inline __attribute__((always_inline)) pmd_t pmd_clear_flags(pmd_t pmd, pmdval_t clear) { pmdval_t v = native_pmd_val(pmd); return __pmd(v & ~clear); } static inline __attribute__((always_inline)) pmd_t pmd_mkold(pmd_t pmd) { return pmd_clear_flags(pmd, (((pteval_t)(1)) << 5)); } static inline __attribute__((always_inline)) pmd_t pmd_wrprotect(pmd_t pmd) { return pmd_clear_flags(pmd, (((pteval_t)(1)) << 1)); } static inline __attribute__((always_inline)) pmd_t pmd_mkdirty(pmd_t pmd) { return pmd_set_flags(pmd, (((pteval_t)(1)) << 6)); } static inline __attribute__((always_inline)) pmd_t pmd_mkhuge(pmd_t pmd) { return pmd_set_flags(pmd, (((pteval_t)(1)) << 7)); } static inline __attribute__((always_inline)) pmd_t pmd_mkyoung(pmd_t pmd) { return pmd_set_flags(pmd, (((pteval_t)(1)) << 5)); } static inline __attribute__((always_inline)) pmd_t pmd_mkwrite(pmd_t pmd) { return pmd_set_flags(pmd, (((pteval_t)(1)) << 1)); } static inline __attribute__((always_inline)) pmd_t pmd_mknotpresent(pmd_t pmd) { return pmd_clear_flags(pmd, (((pteval_t)(1)) << 0)); } static inline __attribute__((always_inline)) pgprotval_t massage_pgprot(pgprot_t pgprot) { pgprotval_t protval = ((pgprot).pgprot); if (__builtin_constant_p(((protval & (((pteval_t)(1)) << 0)))) ? !!((protval & (((pteval_t)(1)) << 0))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/pgtable.h", .line = 301, }; ______r = !!((protval & (((pteval_t)(1)) << 0))); ______f.miss_hit[______r]++; ______r; })) protval &= __supported_pte_mask; return protval; } static inline __attribute__((always_inline)) pte_t pfn_pte(unsigned long page_nr, pgprot_t pgprot) { return __pte(((phys_addr_t)page_nr << 12) | massage_pgprot(pgprot)); } static inline __attribute__((always_inline)) pmd_t pfn_pmd(unsigned long page_nr, pgprot_t pgprot) { return __pmd(((phys_addr_t)page_nr << 12) | massage_pgprot(pgprot)); } static inline __attribute__((always_inline)) pte_t pte_modify(pte_t pte, pgprot_t newprot) { pteval_t val = pte_val(pte); val &= (((pteval_t)(((signed long)(~(((1UL) << 12)-1))) & ((phys_addr_t)((1ULL << 44) - 1)))) | (((pteval_t)(1)) << 4) | (((pteval_t)(1)) << 3) | (((pteval_t)(1)) << 9) | (((pteval_t)(1)) << 5) | (((pteval_t)(1)) << 6)); val |= massage_pgprot(newprot) & ~(((pteval_t)(((signed long)(~(((1UL) << 12)-1))) & ((phys_addr_t)((1ULL << 44) - 1)))) | (((pteval_t)(1)) << 4) | (((pteval_t)(1)) << 3) | (((pteval_t)(1)) << 9) | (((pteval_t)(1)) << 5) | (((pteval_t)(1)) << 6)); return __pte(val); } static inline __attribute__((always_inline)) pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) { pmdval_t val = pmd_val(pmd); val &= ((((pteval_t)(((signed long)(~(((1UL) << 12)-1))) & ((phys_addr_t)((1ULL << 44) - 1)))) | (((pteval_t)(1)) << 4) | (((pteval_t)(1)) << 3) | (((pteval_t)(1)) << 9) | (((pteval_t)(1)) << 5) | (((pteval_t)(1)) << 6)) | (((pteval_t)(1)) << 7)); val |= massage_pgprot(newprot) & ~((((pteval_t)(((signed long)(~(((1UL) << 12)-1))) & ((phys_addr_t)((1ULL << 44) - 1)))) | (((pteval_t)(1)) << 4) | (((pteval_t)(1)) << 3) | (((pteval_t)(1)) << 9) | (((pteval_t)(1)) << 5) | (((pteval_t)(1)) << 6)) | (((pteval_t)(1)) << 7)); return __pmd(val); } static inline __attribute__((always_inline)) pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot) { pgprotval_t preservebits = ((oldprot).pgprot) & (((pteval_t)(((signed long)(~(((1UL) << 12)-1))) & ((phys_addr_t)((1ULL << 44) - 1)))) | (((pteval_t)(1)) << 4) | (((pteval_t)(1)) << 3) | (((pteval_t)(1)) << 9) | (((pteval_t)(1)) << 5) | (((pteval_t)(1)) << 6)); pgprotval_t addbits = ((newprot).pgprot); return ((pgprot_t) { (preservebits | addbits) } ); } static inline __attribute__((always_inline)) int is_new_memtype_allowed(u64 paddr, unsigned long size, unsigned long flags, unsigned long new_flags) { if (__builtin_constant_p(((x86_platform.is_untracked_pat_range(paddr, paddr + size)))) ? !!((x86_platform.is_untracked_pat_range(paddr, paddr + size))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/pgtable.h", .line = 363, }; ______r = !!((x86_platform.is_untracked_pat_range(paddr, paddr + size))); ______f.miss_hit[______r]++; ______r; })) return 1; if (__builtin_constant_p((((flags == ((((pteval_t)(1)) << 4)) && new_flags == (0)) || (flags == ((((pteval_t)(1)) << 3)) && new_flags == (0))))) ? !!(((flags == ((((pteval_t)(1)) << 4)) && new_flags == (0)) || (flags == ((((pteval_t)(1)) << 3)) && new_flags == (0)))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/pgtable.h" , .line = 375 , }; ______r = !!(((flags == ((((pteval_t)(1)) << 4)) && new_flags == (0)) || (flags == ((((pteval_t)(1)) << 3)) && new_flags == (0)))); ______f.miss_hit[______r]++; ______r; })) { return 0; } return 1; } pmd_t *populate_extra_pmd(unsigned long vaddr); pte_t *populate_extra_pte(unsigned long vaddr); struct mm_struct; struct vm_area_struct; extern pgd_t swapper_pg_dir[1024]; extern pgd_t initial_page_table[1024]; static inline __attribute__((always_inline)) void pgtable_cache_init(void) { } static inline __attribute__((always_inline)) void check_pgt_cache(void) { } void paging_init(void); extern void set_pmd_pfn(unsigned long, unsigned long, pgprot_t); static inline __attribute__((always_inline)) void native_set_pte(pte_t *ptep, pte_t pte) { ptep->pte_high = pte.pte_high; __asm__ __volatile__("": : :"memory"); ptep->pte_low = pte.pte_low; } static inline __attribute__((always_inline)) void native_set_pte_atomic(pte_t *ptep, pte_t pte) { set_64bit((unsigned long long *)(ptep), native_pte_val(pte)); } static inline __attribute__((always_inline)) void native_set_pmd(pmd_t *pmdp, pmd_t pmd) { set_64bit((unsigned long long *)(pmdp), native_pmd_val(pmd)); } static inline __attribute__((always_inline)) void native_set_pud(pud_t *pudp, pud_t pud) { set_64bit((unsigned long long *)(pudp), native_pud_val(pud)); } static inline __attribute__((always_inline)) void native_pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { ptep->pte_low = 0; __asm__ __volatile__("": : :"memory"); ptep->pte_high = 0; } static inline __attribute__((always_inline)) void native_pmd_clear(pmd_t *pmd) { u32 *tmp = (u32 *)pmd; *tmp = 0; __asm__ __volatile__("": : :"memory"); *(tmp + 1) = 0; } static inline __attribute__((always_inline)) void pud_clear(pud_t *pudp) { set_pud(pudp, ((pud_t) { __pgd(0) } )); } static inline __attribute__((always_inline)) pte_t native_ptep_get_and_clear(pte_t *ptep) { pte_t res; res.pte_low = ({ __typeof(*((&ptep->pte_low))) __x = ((0)); switch (sizeof(*&ptep->pte_low)) { case 1: { volatile u8 *__ptr = (volatile u8 *)((&ptep->pte_low)); asm volatile("xchgb %0,%1" : "=q" (__x), "+m" (*__ptr) : "0" (__x) : "memory"); break; } case 2: { volatile u16 *__ptr = (volatile u16 *)((&ptep->pte_low)); asm volatile("xchgw %0,%1" : "=r" (__x), "+m" (*__ptr) : "0" (__x) : "memory"); break; } case 4: { volatile u32 *__ptr = (volatile u32 *)((&ptep->pte_low)); asm volatile("xchgl %0,%1" : "=r" (__x), "+m" (*__ptr) : "0" (__x) : "memory"); break; } default: __xchg_wrong_size(); } __x; }); res.pte_high = ptep->pte_high; ptep->pte_high = 0; return res; } union split_pmd { struct { u32 pmd_low; u32 pmd_high; }; pmd_t pmd; }; static inline __attribute__((always_inline)) pmd_t native_pmdp_get_and_clear(pmd_t *pmdp) { union split_pmd res, *orig = (union split_pmd *)pmdp; res.pmd_low = ({ __typeof(*((&orig->pmd_low))) __x = ((0)); switch (sizeof(*&orig->pmd_low)) { case 1: { volatile u8 *__ptr = (volatile u8 *)((&orig->pmd_low)); asm volatile("xchgb %0,%1" : "=q" (__x), "+m" (*__ptr) : "0" (__x) : "memory"); break; } case 2: { volatile u16 *__ptr = (volatile u16 *)((&orig->pmd_low)); asm volatile("xchgw %0,%1" : "=r" (__x), "+m" (*__ptr) : "0" (__x) : "memory"); break; } case 4: { volatile u32 *__ptr = (volatile u32 *)((&orig->pmd_low)); asm volatile("xchgl %0,%1" : "=r" (__x), "+m" (*__ptr) : "0" (__x) : "memory"); break; } default: __xchg_wrong_size(); } __x; }); res.pmd_high = orig->pmd_high; orig->pmd_high = 0; return res.pmd; } static inline __attribute__((always_inline)) int pte_none(pte_t pte) { return !pte.pte; } static inline __attribute__((always_inline)) int pte_same(pte_t a, pte_t b) { return a.pte == b.pte; } static inline __attribute__((always_inline)) int pte_present(pte_t a) { return pte_flags(a) & ((((pteval_t)(1)) << 0) | (((pteval_t)(1)) << 8)); } static inline __attribute__((always_inline)) int pte_hidden(pte_t pte) { return pte_flags(pte) & (((pteval_t)(0))); } static inline __attribute__((always_inline)) int pmd_present(pmd_t pmd) { return pmd_flags(pmd) & (((pteval_t)(1)) << 0); } static inline __attribute__((always_inline)) int pmd_none(pmd_t pmd) { return (unsigned long)native_pmd_val(pmd) == 0; } static inline __attribute__((always_inline)) unsigned long pmd_page_vaddr(pmd_t pmd) { return (unsigned long)((void *)((unsigned long)(pmd_val(pmd) & ((pteval_t)(((signed long)(~(((1UL) << 12)-1))) & ((phys_addr_t)((1ULL << 44) - 1)))))+((unsigned long)(0xC0000000UL)))); } static inline __attribute__((always_inline)) unsigned long pmd_index(unsigned long address) { return (address >> 21) & (512 - 1); } static inline __attribute__((always_inline)) unsigned long pte_index(unsigned long address) { return (address >> 12) & (512 - 1); } static inline __attribute__((always_inline)) pte_t *pte_offset_kernel(pmd_t *pmd, unsigned long address) { return (pte_t *)pmd_page_vaddr(*pmd) + pte_index(address); } static inline __attribute__((always_inline)) int pmd_bad(pmd_t pmd) { return (pmd_flags(pmd) & ~(((pteval_t)(1)) << 2)) != ((((pteval_t)(1)) << 0) | (((pteval_t)(1)) << 1) | (((pteval_t)(1)) << 5) | (((pteval_t)(1)) << 6)); } static inline __attribute__((always_inline)) unsigned long pages_to_mb(unsigned long npg) { return npg >> (20 - 12); } static inline __attribute__((always_inline)) int pud_none(pud_t pud) { return native_pud_val(pud) == 0; } static inline __attribute__((always_inline)) int pud_present(pud_t pud) { return pud_flags(pud) & (((pteval_t)(1)) << 0); } static inline __attribute__((always_inline)) unsigned long pud_page_vaddr(pud_t pud) { return (unsigned long)((void *)((unsigned long)((unsigned long)(pgd_val((pud).pgd)) & ((pteval_t)(((signed long)(~(((1UL) << 12)-1))) & ((phys_addr_t)((1ULL << 44) - 1)))))+((unsigned long)(0xC0000000UL)))); } static inline __attribute__((always_inline)) pmd_t *pmd_offset(pud_t *pud, unsigned long address) { return (pmd_t *)pud_page_vaddr(*pud) + pmd_index(address); } static inline __attribute__((always_inline)) int pud_large(pud_t pud) { return ((pgd_val((pud).pgd)) & ((((pteval_t)(1)) << 7) | (((pteval_t)(1)) << 0))) == ((((pteval_t)(1)) << 7) | (((pteval_t)(1)) << 0)); } static inline __attribute__((always_inline)) int pud_bad(pud_t pud) { return (pud_flags(pud) & ~(((((pteval_t)(1)) << 0) | (((pteval_t)(1)) << 1) | (((pteval_t)(1)) << 5) | (((pteval_t)(1)) << 6)) | (((pteval_t)(1)) << 2))) != 0; } extern int direct_gbpages; static inline __attribute__((always_inline)) pte_t native_local_ptep_get_and_clear(pte_t *ptep) { pte_t res = *ptep; native_pte_clear(((void *)0), 0, ptep); return res; } static inline __attribute__((always_inline)) pmd_t native_local_pmdp_get_and_clear(pmd_t *pmdp) { pmd_t res = *pmdp; native_pmd_clear(pmdp); return res; } static inline __attribute__((always_inline)) void native_set_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep , pte_t pte) { native_set_pte(ptep, pte); } static inline __attribute__((always_inline)) void native_set_pmd_at(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp , pmd_t pmd) { native_set_pmd(pmdp, pmd); } struct vm_area_struct; extern int ptep_set_access_flags(struct vm_area_struct *vma, unsigned long address, pte_t *ptep, pte_t entry, int dirty); extern int ptep_test_and_clear_young(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep); extern int ptep_clear_flush_young(struct vm_area_struct *vma, unsigned long address, pte_t *ptep); static inline __attribute__((always_inline)) pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { pte_t pte = native_ptep_get_and_clear(ptep); pte_update(mm, addr, ptep); return pte; } static inline __attribute__((always_inline)) pte_t ptep_get_and_clear_full(struct mm_struct *mm, unsigned long addr, pte_t *ptep, int full) { pte_t pte; if (__builtin_constant_p(((full))) ? !!((full)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/pgtable.h", .line = 686, }; ______r = !!((full)); ______f.miss_hit[______r]++; ______r; })) { pte = native_local_ptep_get_and_clear(ptep); } else { pte = ptep_get_and_clear(mm, addr, ptep); } return pte; } static inline __attribute__((always_inline)) void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { clear_bit(1, (unsigned long *)&ptep->pte); pte_update(mm, addr, ptep); } extern int pmdp_set_access_flags(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp, pmd_t entry, int dirty); extern int pmdp_test_and_clear_young(struct vm_area_struct *vma, unsigned long addr, pmd_t *pmdp); extern int pmdp_clear_flush_young(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp); extern void pmdp_splitting_flush(struct vm_area_struct *vma, unsigned long addr, pmd_t *pmdp); static inline __attribute__((always_inline)) int pmd_write(pmd_t pmd) { return pmd_flags(pmd) & (((pteval_t)(1)) << 1); } static inline __attribute__((always_inline)) pmd_t pmdp_get_and_clear(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp) { pmd_t pmd = native_pmdp_get_and_clear(pmdp); pmd_update(mm, addr, pmdp); return pmd; } static inline __attribute__((always_inline)) void pmdp_set_wrprotect(struct mm_struct *mm, unsigned long addr, pmd_t *pmdp) { clear_bit(1, (unsigned long *)pmdp); pmd_update(mm, addr, pmdp); } static inline __attribute__((always_inline)) void clone_pgd_range(pgd_t *dst, pgd_t *src, int count) { __builtin_memcpy(dst, src, count * sizeof(pgd_t)); } static inline __attribute__((always_inline)) void pte_clear_not_present_full(struct mm_struct *mm, unsigned long address, pte_t *ptep, int full) { pte_clear(mm, address, ptep); } extern pte_t ptep_clear_flush(struct vm_area_struct *vma, unsigned long address, pte_t *ptep); extern pmd_t pmdp_clear_flush(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp); static inline __attribute__((always_inline)) int pmd_same(pmd_t pmd_a, pmd_t pmd_b) { return pmd_val(pmd_a) == pmd_val(pmd_b); } void pgd_clear_bad(pgd_t *); void pud_clear_bad(pud_t *); void pmd_clear_bad(pmd_t *); static inline __attribute__((always_inline)) int pgd_none_or_clear_bad(pgd_t *pgd) { if (__builtin_constant_p(((pgd_none(*pgd)))) ? !!((pgd_none(*pgd))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/asm-generic/pgtable.h", .line = 257, }; ______r = !!((pgd_none(*pgd))); ______f.miss_hit[______r]++; ______r; })) return 1; if (__builtin_constant_p((((__builtin_constant_p(pgd_bad(*pgd)) ? !!(pgd_bad(*pgd)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/asm-generic/pgtable.h", .line = 259, }; ______r = __builtin_expect(!!(pgd_bad(*pgd)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(pgd_bad(*pgd)) ? !!(pgd_bad(*pgd)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/asm-generic/pgtable.h", .line = 259, }; ______r = __builtin_expect(!!(pgd_bad(*pgd)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/asm-generic/pgtable.h", .line = 259, }; ______r = !!(((__builtin_constant_p(pgd_bad(*pgd)) ? !!(pgd_bad(*pgd)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/asm-generic/pgtable.h", .line = 259, }; ______r = __builtin_expect(!!(pgd_bad(*pgd)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) { pgd_clear_bad(pgd); return 1; } return 0; } static inline __attribute__((always_inline)) int pud_none_or_clear_bad(pud_t *pud) { if (__builtin_constant_p(((pud_none(*pud)))) ? !!((pud_none(*pud))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/asm-generic/pgtable.h", .line = 268, }; ______r = !!((pud_none(*pud))); ______f.miss_hit[______r]++; ______r; })) return 1; if (__builtin_constant_p((((__builtin_constant_p(pud_bad(*pud)) ? !!(pud_bad(*pud)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/asm-generic/pgtable.h", .line = 270, }; ______r = __builtin_expect(!!(pud_bad(*pud)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(pud_bad(*pud)) ? !!(pud_bad(*pud)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/asm-generic/pgtable.h", .line = 270, }; ______r = __builtin_expect(!!(pud_bad(*pud)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/asm-generic/pgtable.h", .line = 270, }; ______r = !!(((__builtin_constant_p(pud_bad(*pud)) ? !!(pud_bad(*pud)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/asm-generic/pgtable.h", .line = 270, }; ______r = __builtin_expect(!!(pud_bad(*pud)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) { pud_clear_bad(pud); return 1; } return 0; } static inline __attribute__((always_inline)) int pmd_none_or_clear_bad(pmd_t *pmd) { if (__builtin_constant_p(((pmd_none(*pmd)))) ? !!((pmd_none(*pmd))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/asm-generic/pgtable.h", .line = 279, }; ______r = !!((pmd_none(*pmd))); ______f.miss_hit[______r]++; ______r; })) return 1; if (__builtin_constant_p((((__builtin_constant_p(pmd_bad(*pmd)) ? !!(pmd_bad(*pmd)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/asm-generic/pgtable.h", .line = 281, }; ______r = __builtin_expect(!!(pmd_bad(*pmd)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(pmd_bad(*pmd)) ? !!(pmd_bad(*pmd)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/asm-generic/pgtable.h", .line = 281, }; ______r = __builtin_expect(!!(pmd_bad(*pmd)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/asm-generic/pgtable.h", .line = 281, }; ______r = !!(((__builtin_constant_p(pmd_bad(*pmd)) ? !!(pmd_bad(*pmd)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/asm-generic/pgtable.h", .line = 281, }; ______r = __builtin_expect(!!(pmd_bad(*pmd)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) { pmd_clear_bad(pmd); return 1; } return 0; } static inline __attribute__((always_inline)) pte_t __ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { return ptep_get_and_clear(mm, addr, ptep); } static inline __attribute__((always_inline)) void __ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t pte) { set_pte_at(mm, addr, ptep, pte); } extern int track_pfn_vma_new(struct vm_area_struct *vma, pgprot_t *prot, unsigned long pfn, unsigned long size); extern int track_pfn_vma_copy(struct vm_area_struct *vma); extern void untrack_pfn_vma(struct vm_area_struct *vma, unsigned long pfn, unsigned long size); extern struct kmem_cache *vm_area_cachep; extern pgprot_t protection_map[16]; static inline __attribute__((always_inline)) int is_linear_pfn_mapping(struct vm_area_struct *vma) { return !!(vma->vm_flags & 0x40000000); } static inline __attribute__((always_inline)) int is_pfn_mapping(struct vm_area_struct *vma) { return !!(vma->vm_flags & 0x00000400); } struct vm_fault { unsigned int flags; unsigned long pgoff; void *virtual_address; struct page *page; }; struct vm_operations_struct { void (*open)(struct vm_area_struct * area); void (*close)(struct vm_area_struct * area); int (*fault)(struct vm_area_struct *vma, struct vm_fault *vmf); int (*page_mkwrite)(struct vm_area_struct *vma, struct vm_fault *vmf); int (*access)(struct vm_area_struct *vma, unsigned long addr, void *buf, int len, int write); }; struct mmu_gather; struct inode; enum pageflags { PG_locked, PG_error, PG_referenced, PG_uptodate, PG_dirty, PG_lru, PG_active, PG_slab, PG_owner_priv_1, PG_arch_1, PG_reserved, PG_private, PG_private_2, PG_writeback, PG_head, PG_tail, PG_swapcache, PG_mappedtodisk, PG_reclaim, PG_swapbacked, PG_unevictable, PG_mlocked, PG_uncached, PG_compound_lock, __NR_PAGEFLAGS, PG_checked = PG_owner_priv_1, PG_fscache = PG_private_2, PG_pinned = PG_owner_priv_1, PG_savepinned = PG_dirty, PG_slob_free = PG_private, PG_slub_frozen = PG_active, }; struct page; static inline __attribute__((always_inline)) int PageLocked(struct page *page) { return (__builtin_constant_p((PG_locked)) ? constant_test_bit((PG_locked), (&page->flags)) : variable_test_bit((PG_locked), (&page->flags))); } static inline __attribute__((always_inline)) int PageError(struct page *page) { return (__builtin_constant_p((PG_error)) ? constant_test_bit((PG_error), (&page->flags)) : variable_test_bit((PG_error), (&page->flags))); } static inline __attribute__((always_inline)) void SetPageError(struct page *page) { set_bit(PG_error, &page->flags); } static inline __attribute__((always_inline)) void ClearPageError(struct page *page) { clear_bit(PG_error, &page->flags); } static inline __attribute__((always_inline)) int TestClearPageError(struct page *page) { return test_and_clear_bit(PG_error, &page->flags); } static inline __attribute__((always_inline)) int PageReferenced(struct page *page) { return (__builtin_constant_p((PG_referenced)) ? constant_test_bit((PG_referenced), (&page->flags)) : variable_test_bit((PG_referenced), (&page->flags))); } static inline __attribute__((always_inline)) void SetPageReferenced(struct page *page) { set_bit(PG_referenced, &page->flags); } static inline __attribute__((always_inline)) void ClearPageReferenced(struct page *page) { clear_bit(PG_referenced, &page->flags); } static inline __attribute__((always_inline)) int TestClearPageReferenced(struct page *page) { return test_and_clear_bit(PG_referenced, &page->flags); } static inline __attribute__((always_inline)) int PageDirty(struct page *page) { return (__builtin_constant_p((PG_dirty)) ? constant_test_bit((PG_dirty), (&page->flags)) : variable_test_bit((PG_dirty), (&page->flags))); } static inline __attribute__((always_inline)) void SetPageDirty(struct page *page) { set_bit(PG_dirty, &page->flags); } static inline __attribute__((always_inline)) void ClearPageDirty(struct page *page) { clear_bit(PG_dirty, &page->flags); } static inline __attribute__((always_inline)) int TestSetPageDirty(struct page *page) { return test_and_set_bit(PG_dirty, &page->flags); } static inline __attribute__((always_inline)) int TestClearPageDirty(struct page *page) { return test_and_clear_bit(PG_dirty, &page->flags); } static inline __attribute__((always_inline)) void __ClearPageDirty(struct page *page) { __clear_bit(PG_dirty, &page->flags); } static inline __attribute__((always_inline)) int PageLRU(struct page *page) { return (__builtin_constant_p((PG_lru)) ? constant_test_bit((PG_lru), (&page->flags)) : variable_test_bit((PG_lru), (&page->flags))); } static inline __attribute__((always_inline)) void SetPageLRU(struct page *page) { set_bit(PG_lru, &page->flags); } static inline __attribute__((always_inline)) void ClearPageLRU(struct page *page) { clear_bit(PG_lru, &page->flags); } static inline __attribute__((always_inline)) void __ClearPageLRU(struct page *page) { __clear_bit(PG_lru, &page->flags); } static inline __attribute__((always_inline)) int PageActive(struct page *page) { return (__builtin_constant_p((PG_active)) ? constant_test_bit((PG_active), (&page->flags)) : variable_test_bit((PG_active), (&page->flags))); } static inline __attribute__((always_inline)) void SetPageActive(struct page *page) { set_bit(PG_active, &page->flags); } static inline __attribute__((always_inline)) void ClearPageActive(struct page *page) { clear_bit(PG_active, &page->flags); } static inline __attribute__((always_inline)) void __ClearPageActive(struct page *page) { __clear_bit(PG_active, &page->flags); } static inline __attribute__((always_inline)) int TestClearPageActive(struct page *page) { return test_and_clear_bit(PG_active, &page->flags); } static inline __attribute__((always_inline)) int PageSlab(struct page *page) { return (__builtin_constant_p((PG_slab)) ? constant_test_bit((PG_slab), (&page->flags)) : variable_test_bit((PG_slab), (&page->flags))); } static inline __attribute__((always_inline)) void __SetPageSlab(struct page *page) { __set_bit(PG_slab, &page->flags); } static inline __attribute__((always_inline)) void __ClearPageSlab(struct page *page) { __clear_bit(PG_slab, &page->flags); } static inline __attribute__((always_inline)) int PageChecked(struct page *page) { return (__builtin_constant_p((PG_checked)) ? constant_test_bit((PG_checked), (&page->flags)) : variable_test_bit((PG_checked), (&page->flags))); } static inline __attribute__((always_inline)) void SetPageChecked(struct page *page) { set_bit(PG_checked, &page->flags); } static inline __attribute__((always_inline)) void ClearPageChecked(struct page *page) { clear_bit(PG_checked, &page->flags); } static inline __attribute__((always_inline)) int PagePinned(struct page *page) { return (__builtin_constant_p((PG_pinned)) ? constant_test_bit((PG_pinned), (&page->flags)) : variable_test_bit((PG_pinned), (&page->flags))); } static inline __attribute__((always_inline)) void SetPagePinned(struct page *page) { set_bit(PG_pinned, &page->flags); } static inline __attribute__((always_inline)) void ClearPagePinned(struct page *page) { clear_bit(PG_pinned, &page->flags); } static inline __attribute__((always_inline)) int TestSetPagePinned(struct page *page) { return test_and_set_bit(PG_pinned, &page->flags); } static inline __attribute__((always_inline)) int TestClearPagePinned(struct page *page) { return test_and_clear_bit(PG_pinned, &page->flags); } static inline __attribute__((always_inline)) int PageSavePinned(struct page *page) { return (__builtin_constant_p((PG_savepinned)) ? constant_test_bit((PG_savepinned), (&page->flags)) : variable_test_bit((PG_savepinned), (&page->flags))); } static inline __attribute__((always_inline)) void SetPageSavePinned(struct page *page) { set_bit(PG_savepinned, &page->flags); } static inline __attribute__((always_inline)) void ClearPageSavePinned(struct page *page) { clear_bit(PG_savepinned, &page->flags); }; static inline __attribute__((always_inline)) int PageReserved(struct page *page) { return (__builtin_constant_p((PG_reserved)) ? constant_test_bit((PG_reserved), (&page->flags)) : variable_test_bit((PG_reserved), (&page->flags))); } static inline __attribute__((always_inline)) void SetPageReserved(struct page *page) { set_bit(PG_reserved, &page->flags); } static inline __attribute__((always_inline)) void ClearPageReserved(struct page *page) { clear_bit(PG_reserved, &page->flags); } static inline __attribute__((always_inline)) void __ClearPageReserved(struct page *page) { __clear_bit(PG_reserved, &page->flags); } static inline __attribute__((always_inline)) int PageSwapBacked(struct page *page) { return (__builtin_constant_p((PG_swapbacked)) ? constant_test_bit((PG_swapbacked), (&page->flags)) : variable_test_bit((PG_swapbacked), (&page->flags))); } static inline __attribute__((always_inline)) void SetPageSwapBacked(struct page *page) { set_bit(PG_swapbacked, &page->flags); } static inline __attribute__((always_inline)) void ClearPageSwapBacked(struct page *page) { clear_bit(PG_swapbacked, &page->flags); } static inline __attribute__((always_inline)) void __ClearPageSwapBacked(struct page *page) { __clear_bit(PG_swapbacked, &page->flags); } static inline __attribute__((always_inline)) int PageSlobFree(struct page *page) { return (__builtin_constant_p((PG_slob_free)) ? constant_test_bit((PG_slob_free), (&page->flags)) : variable_test_bit((PG_slob_free), (&page->flags))); } static inline __attribute__((always_inline)) void __SetPageSlobFree(struct page *page) { __set_bit(PG_slob_free, &page->flags); } static inline __attribute__((always_inline)) void __ClearPageSlobFree(struct page *page) { __clear_bit(PG_slob_free, &page->flags); } static inline __attribute__((always_inline)) int PageSlubFrozen(struct page *page) { return (__builtin_constant_p((PG_slub_frozen)) ? constant_test_bit((PG_slub_frozen), (&page->flags)) : variable_test_bit((PG_slub_frozen), (&page->flags))); } static inline __attribute__((always_inline)) void __SetPageSlubFrozen(struct page *page) { __set_bit(PG_slub_frozen, &page->flags); } static inline __attribute__((always_inline)) void __ClearPageSlubFrozen(struct page *page) { __clear_bit(PG_slub_frozen, &page->flags); } static inline __attribute__((always_inline)) int PagePrivate(struct page *page) { return (__builtin_constant_p((PG_private)) ? constant_test_bit((PG_private), (&page->flags)) : variable_test_bit((PG_private), (&page->flags))); } static inline __attribute__((always_inline)) void SetPagePrivate(struct page *page) { set_bit(PG_private, &page->flags); } static inline __attribute__((always_inline)) void ClearPagePrivate(struct page *page) { clear_bit(PG_private, &page->flags); } static inline __attribute__((always_inline)) void __SetPagePrivate(struct page *page) { __set_bit(PG_private, &page->flags); } static inline __attribute__((always_inline)) void __ClearPagePrivate(struct page *page) { __clear_bit(PG_private, &page->flags); } static inline __attribute__((always_inline)) int PagePrivate2(struct page *page) { return (__builtin_constant_p((PG_private_2)) ? constant_test_bit((PG_private_2), (&page->flags)) : variable_test_bit((PG_private_2), (&page->flags))); } static inline __attribute__((always_inline)) void SetPagePrivate2(struct page *page) { set_bit(PG_private_2, &page->flags); } static inline __attribute__((always_inline)) void ClearPagePrivate2(struct page *page) { clear_bit(PG_private_2, &page->flags); } static inline __attribute__((always_inline)) int TestSetPagePrivate2(struct page *page) { return test_and_set_bit(PG_private_2, &page->flags); } static inline __attribute__((always_inline)) int TestClearPagePrivate2(struct page *page) { return test_and_clear_bit(PG_private_2, &page->flags); } static inline __attribute__((always_inline)) int PageOwnerPriv1(struct page *page) { return (__builtin_constant_p((PG_owner_priv_1)) ? constant_test_bit((PG_owner_priv_1), (&page->flags)) : variable_test_bit((PG_owner_priv_1), (&page->flags))); } static inline __attribute__((always_inline)) void SetPageOwnerPriv1(struct page *page) { set_bit(PG_owner_priv_1, &page->flags); } static inline __attribute__((always_inline)) void ClearPageOwnerPriv1(struct page *page) { clear_bit(PG_owner_priv_1, &page->flags); } static inline __attribute__((always_inline)) int TestClearPageOwnerPriv1(struct page *page) { return test_and_clear_bit(PG_owner_priv_1, &page->flags); } static inline __attribute__((always_inline)) int PageWriteback(struct page *page) { return (__builtin_constant_p((PG_writeback)) ? constant_test_bit((PG_writeback), (&page->flags)) : variable_test_bit((PG_writeback), (&page->flags))); } static inline __attribute__((always_inline)) int TestSetPageWriteback(struct page *page) { return test_and_set_bit(PG_writeback, &page->flags); } static inline __attribute__((always_inline)) int TestClearPageWriteback(struct page *page) { return test_and_clear_bit(PG_writeback, &page->flags); } static inline __attribute__((always_inline)) int PageMappedToDisk(struct page *page) { return (__builtin_constant_p((PG_mappedtodisk)) ? constant_test_bit((PG_mappedtodisk), (&page->flags)) : variable_test_bit((PG_mappedtodisk), (&page->flags))); } static inline __attribute__((always_inline)) void SetPageMappedToDisk(struct page *page) { set_bit(PG_mappedtodisk, &page->flags); } static inline __attribute__((always_inline)) void ClearPageMappedToDisk(struct page *page) { clear_bit(PG_mappedtodisk, &page->flags); } static inline __attribute__((always_inline)) int PageReclaim(struct page *page) { return (__builtin_constant_p((PG_reclaim)) ? constant_test_bit((PG_reclaim), (&page->flags)) : variable_test_bit((PG_reclaim), (&page->flags))); } static inline __attribute__((always_inline)) void SetPageReclaim(struct page *page) { set_bit(PG_reclaim, &page->flags); } static inline __attribute__((always_inline)) void ClearPageReclaim(struct page *page) { clear_bit(PG_reclaim, &page->flags); } static inline __attribute__((always_inline)) int TestClearPageReclaim(struct page *page) { return test_and_clear_bit(PG_reclaim, &page->flags); } static inline __attribute__((always_inline)) int PageReadahead(struct page *page) { return (__builtin_constant_p((PG_reclaim)) ? constant_test_bit((PG_reclaim), (&page->flags)) : variable_test_bit((PG_reclaim), (&page->flags))); } static inline __attribute__((always_inline)) void SetPageReadahead(struct page *page) { set_bit(PG_reclaim, &page->flags); } static inline __attribute__((always_inline)) void ClearPageReadahead(struct page *page) { clear_bit(PG_reclaim, &page->flags); } static inline __attribute__((always_inline)) int PageSwapCache(struct page *page) { return (__builtin_constant_p((PG_swapcache)) ? constant_test_bit((PG_swapcache), (&page->flags)) : variable_test_bit((PG_swapcache), (&page->flags))); } static inline __attribute__((always_inline)) void SetPageSwapCache(struct page *page) { set_bit(PG_swapcache, &page->flags); } static inline __attribute__((always_inline)) void ClearPageSwapCache(struct page *page) { clear_bit(PG_swapcache, &page->flags); } static inline __attribute__((always_inline)) int PageUnevictable(struct page *page) { return (__builtin_constant_p((PG_unevictable)) ? constant_test_bit((PG_unevictable), (&page->flags)) : variable_test_bit((PG_unevictable), (&page->flags))); } static inline __attribute__((always_inline)) void SetPageUnevictable(struct page *page) { set_bit(PG_unevictable, &page->flags); } static inline __attribute__((always_inline)) void ClearPageUnevictable(struct page *page) { clear_bit(PG_unevictable, &page->flags); } static inline __attribute__((always_inline)) void __ClearPageUnevictable(struct page *page) { __clear_bit(PG_unevictable, &page->flags); } static inline __attribute__((always_inline)) int TestClearPageUnevictable(struct page *page) { return test_and_clear_bit(PG_unevictable, &page->flags); } static inline __attribute__((always_inline)) int PageMlocked(struct page *page) { return (__builtin_constant_p((PG_mlocked)) ? constant_test_bit((PG_mlocked), (&page->flags)) : variable_test_bit((PG_mlocked), (&page->flags))); } static inline __attribute__((always_inline)) void SetPageMlocked(struct page *page) { set_bit(PG_mlocked, &page->flags); } static inline __attribute__((always_inline)) void ClearPageMlocked(struct page *page) { clear_bit(PG_mlocked, &page->flags); } static inline __attribute__((always_inline)) void __ClearPageMlocked(struct page *page) { __clear_bit(PG_mlocked, &page->flags); } static inline __attribute__((always_inline)) int TestSetPageMlocked(struct page *page) { return test_and_set_bit(PG_mlocked, &page->flags); } static inline __attribute__((always_inline)) int TestClearPageMlocked(struct page *page) { return test_and_clear_bit(PG_mlocked, &page->flags); } static inline __attribute__((always_inline)) int __TestClearPageMlocked(struct page *page) { return __test_and_clear_bit(PG_mlocked, &page->flags); } static inline __attribute__((always_inline)) int PageUncached(struct page *page) { return (__builtin_constant_p((PG_uncached)) ? constant_test_bit((PG_uncached), (&page->flags)) : variable_test_bit((PG_uncached), (&page->flags))); } static inline __attribute__((always_inline)) void SetPageUncached(struct page *page) { set_bit(PG_uncached, &page->flags); } static inline __attribute__((always_inline)) void ClearPageUncached(struct page *page) { clear_bit(PG_uncached, &page->flags); } static inline __attribute__((always_inline)) int PageHWPoison(struct page *page) { return 0; } u64 stable_page_flags(struct page *page); static inline __attribute__((always_inline)) int PageUptodate(struct page *page) { int ret = (__builtin_constant_p((PG_uptodate)) ? constant_test_bit((PG_uptodate), (&(page)->flags)) : variable_test_bit((PG_uptodate), (&(page)->flags))); if (__builtin_constant_p(((ret))) ? !!((ret)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/page-flags.h", .line = 295, }; ______r = !!((ret)); ______f.miss_hit[______r]++; ______r; })) __asm__ __volatile__("": : :"memory"); return ret; } static inline __attribute__((always_inline)) void __SetPageUptodate(struct page *page) { __asm__ __volatile__("": : :"memory"); __set_bit(PG_uptodate, &(page)->flags); } static inline __attribute__((always_inline)) void SetPageUptodate(struct page *page) { __asm__ __volatile__("": : :"memory"); set_bit(PG_uptodate, &(page)->flags); } static inline __attribute__((always_inline)) void ClearPageUptodate(struct page *page) { clear_bit(PG_uptodate, &page->flags); } extern void cancel_dirty_page(struct page *page, unsigned int account_size); int test_clear_page_writeback(struct page *page); int test_set_page_writeback(struct page *page); static inline __attribute__((always_inline)) void set_page_writeback(struct page *page) { test_set_page_writeback(page); } static inline __attribute__((always_inline)) int PageHead(struct page *page) { return (__builtin_constant_p((PG_head)) ? constant_test_bit((PG_head), (&page->flags)) : variable_test_bit((PG_head), (&page->flags))); } static inline __attribute__((always_inline)) void __SetPageHead(struct page *page) { __set_bit(PG_head, &page->flags); } static inline __attribute__((always_inline)) void __ClearPageHead(struct page *page) { __clear_bit(PG_head, &page->flags); } static inline __attribute__((always_inline)) void ClearPageHead(struct page *page) { clear_bit(PG_head, &page->flags); } static inline __attribute__((always_inline)) int PageTail(struct page *page) { return (__builtin_constant_p((PG_tail)) ? constant_test_bit((PG_tail), (&page->flags)) : variable_test_bit((PG_tail), (&page->flags))); } static inline __attribute__((always_inline)) void __SetPageTail(struct page *page) { __set_bit(PG_tail, &page->flags); } static inline __attribute__((always_inline)) void __ClearPageTail(struct page *page) { __clear_bit(PG_tail, &page->flags); } static inline __attribute__((always_inline)) int PageCompound(struct page *page) { return page->flags & ((1L << PG_head) | (1L << PG_tail)); } static inline __attribute__((always_inline)) void ClearPageCompound(struct page *page) { do { if (__builtin_constant_p((((__builtin_constant_p(!PageHead(page)) ? !!(!PageHead(page)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/page-flags.h", .line = 356, }; ______r = __builtin_expect(!!(!PageHead(page)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(!PageHead(page)) ? !!(!PageHead(page)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/page-flags.h", .line = 356, }; ______r = __builtin_expect(!!(!PageHead(page)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/page-flags.h", .line = 356, }; ______r = !!(((__builtin_constant_p(!PageHead(page)) ? !!(!PageHead(page)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/page-flags.h", .line = 356, }; ______r = __builtin_expect(!!(!PageHead(page)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) do { asm volatile("1:\tud2\n" ".pushsection __bug_table,\"a\"\n" "2:\t.long 1b, %c0\n" "\t.word %c1, 0\n" "\t.org 2b+%c2\n" ".popsection" : : "i" ("include/linux/page-flags.h"), "i" (356), "i" (sizeof(struct bug_entry))); __builtin_unreachable(); } while (0); } while(0); ClearPageHead(page); } static inline __attribute__((always_inline)) int PageTransHuge(struct page *page) { do { (void)(PageTail(page)); } while (0); return PageHead(page); } static inline __attribute__((always_inline)) int PageTransCompound(struct page *page) { return PageCompound(page); } static inline __attribute__((always_inline)) int page_has_private(struct page *page) { return !!(page->flags & (1 << PG_private | 1 << PG_private_2)); } extern int do_huge_pmd_anonymous_page(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long address, pmd_t *pmd, unsigned int flags); extern int copy_huge_pmd(struct mm_struct *dst_mm, struct mm_struct *src_mm, pmd_t *dst_pmd, pmd_t *src_pmd, unsigned long addr, struct vm_area_struct *vma); extern int do_huge_pmd_wp_page(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long address, pmd_t *pmd, pmd_t orig_pmd); extern pgtable_t get_pmd_huge_pte(struct mm_struct *mm); extern struct page *follow_trans_huge_pmd(struct mm_struct *mm, unsigned long addr, pmd_t *pmd, unsigned int flags); extern int zap_huge_pmd(struct mmu_gather *tlb, struct vm_area_struct *vma, pmd_t *pmd); extern int mincore_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr, unsigned long end, unsigned char *vec); extern int change_huge_pmd(struct vm_area_struct *vma, pmd_t *pmd, unsigned long addr, pgprot_t newprot); enum transparent_hugepage_flag { TRANSPARENT_HUGEPAGE_FLAG, TRANSPARENT_HUGEPAGE_REQ_MADV_FLAG, TRANSPARENT_HUGEPAGE_DEFRAG_FLAG, TRANSPARENT_HUGEPAGE_DEFRAG_REQ_MADV_FLAG, TRANSPARENT_HUGEPAGE_DEFRAG_KHUGEPAGED_FLAG, }; enum page_check_address_pmd_flag { PAGE_CHECK_ADDRESS_PMD_FLAG, PAGE_CHECK_ADDRESS_PMD_NOTSPLITTING_FLAG, PAGE_CHECK_ADDRESS_PMD_SPLITTING_FLAG, }; extern pmd_t *page_check_address_pmd(struct page *page, struct mm_struct *mm, unsigned long address, enum page_check_address_pmd_flag flag); extern unsigned long transparent_hugepage_flags; extern int copy_pte_range(struct mm_struct *dst_mm, struct mm_struct *src_mm, pmd_t *dst_pmd, pmd_t *src_pmd, struct vm_area_struct *vma, unsigned long addr, unsigned long end); extern int handle_pte_fault(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long address, pte_t *pte, pmd_t *pmd, unsigned int flags); extern int split_huge_page(struct page *page); extern void __split_huge_page_pmd(struct mm_struct *mm, pmd_t *pmd); extern int hugepage_madvise(struct vm_area_struct *vma, unsigned long *vm_flags, int advice); extern void __vma_adjust_trans_huge(struct vm_area_struct *vma, unsigned long start, unsigned long end, long adjust_next); static inline __attribute__((always_inline)) void vma_adjust_trans_huge(struct vm_area_struct *vma, unsigned long start, unsigned long end, long adjust_next) { if (__builtin_constant_p(((!vma->anon_vma || vma->vm_ops))) ? !!((!vma->anon_vma || vma->vm_ops)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/huge_mm.h", .line = 116, }; ______r = !!((!vma->anon_vma || vma->vm_ops)); ______f.miss_hit[______r]++; ______r; })) return; __vma_adjust_trans_huge(vma, start, end, adjust_next); } static inline __attribute__((always_inline)) int hpage_nr_pages(struct page *page) { if (__builtin_constant_p((((__builtin_constant_p(PageTransHuge(page)) ? !!(PageTransHuge(page)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/huge_mm.h", .line = 122, }; ______r = __builtin_expect(!!(PageTransHuge(page)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(PageTransHuge(page)) ? !!(PageTransHuge(page)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/huge_mm.h", .line = 122, }; ______r = __builtin_expect(!!(PageTransHuge(page)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/huge_mm.h", .line = 122, }; ______r = !!(((__builtin_constant_p(PageTransHuge(page)) ? !!(PageTransHuge(page)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/huge_mm.h", .line = 122, }; ______r = __builtin_expect(!!(PageTransHuge(page)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) return (1<<(21 -12)); return 1; } static inline __attribute__((always_inline)) struct page *compound_trans_head(struct page *page) { if (__builtin_constant_p(((PageTail(page)))) ? !!((PageTail(page))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/huge_mm.h", .line = 128, }; ______r = !!((PageTail(page))); ______f.miss_hit[______r]++; ______r; })) { struct page *head; head = page->first_page; __asm__ __volatile__("": : :"memory"); if (__builtin_constant_p(((PageTail(page)))) ? !!((PageTail(page))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/huge_mm.h", .line = 138, }; ______r = !!((PageTail(page))); ______f.miss_hit[______r]++; ______r; })) return head; } return page; } static inline __attribute__((always_inline)) int put_page_testzero(struct page *page) { do { (void)(atomic_read(&page->_count) == 0); } while (0); return atomic_dec_and_test(&page->_count); } static inline __attribute__((always_inline)) int get_page_unless_zero(struct page *page) { return atomic_add_unless((&page->_count), 1, 0); } extern int page_is_ram(unsigned long pfn); struct page *vmalloc_to_page(const void *addr); unsigned long vmalloc_to_pfn(const void *addr); static inline __attribute__((always_inline)) int is_vmalloc_addr(const void *x) { unsigned long addr = (unsigned long)x; return addr >= ((unsigned long)high_memory + (8 * 1024 * 1024)) && addr < ((((((unsigned long)__FIXADDR_TOP) - (__end_of_fixed_addresses << 12)) - ((1UL) << 12) * (512 + 1)) & (~((1UL << 21) - 1))) - 2 * ((1UL) << 12)); } extern int is_vmalloc_or_module_addr(const void *x); static inline __attribute__((always_inline)) void compound_lock(struct page *page) { bit_spin_lock(PG_compound_lock, &page->flags); } static inline __attribute__((always_inline)) void compound_unlock(struct page *page) { bit_spin_unlock(PG_compound_lock, &page->flags); } static inline __attribute__((always_inline)) unsigned long compound_lock_irqsave(struct page *page) { unsigned long flags = flags; do { do { ({ unsigned long __dummy; typeof(flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); flags = arch_local_irq_save(); } while (0); trace_hardirqs_off(); } while (0); compound_lock(page); return flags; } static inline __attribute__((always_inline)) void compound_unlock_irqrestore(struct page *page, unsigned long flags) { compound_unlock(page); do { if (__builtin_constant_p(((({ ({ unsigned long __dummy; typeof(flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); arch_irqs_disabled_flags(flags); })))) ? !!((({ ({ unsigned long __dummy; typeof(flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); arch_irqs_disabled_flags(flags); }))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/mm.h", .line = 347, }; ______r = !!((({ ({ unsigned long __dummy; typeof(flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); arch_irqs_disabled_flags(flags); }))); ______f.miss_hit[______r]++; ______r; })) { do { ({ unsigned long __dummy; typeof(flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); arch_local_irq_restore(flags); } while (0); trace_hardirqs_off(); } else { trace_hardirqs_on(); do { ({ unsigned long __dummy; typeof(flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); arch_local_irq_restore(flags); } while (0); } } while (0); } static inline __attribute__((always_inline)) struct page *compound_head(struct page *page) { if (__builtin_constant_p((((__builtin_constant_p(PageTail(page)) ? !!(PageTail(page)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/mm.h", .line = 353, }; ______r = __builtin_expect(!!(PageTail(page)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(PageTail(page)) ? !!(PageTail(page)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/mm.h", .line = 353, }; ______r = __builtin_expect(!!(PageTail(page)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/mm.h", .line = 353, }; ______r = !!(((__builtin_constant_p(PageTail(page)) ? !!(PageTail(page)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/mm.h", .line = 353, }; ______r = __builtin_expect(!!(PageTail(page)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) return page->first_page; return page; } static inline __attribute__((always_inline)) int page_count(struct page *page) { return atomic_read(&compound_head(page)->_count); } static inline __attribute__((always_inline)) void get_page(struct page *page) { do { (void)(atomic_read(&page->_count) < !PageTail(page)); } while (0); atomic_inc(&page->_count); if (__builtin_constant_p((((__builtin_constant_p(PageTail(page)) ? !!(PageTail(page)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/mm.h", .line = 379, }; ______r = __builtin_expect(!!(PageTail(page)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(PageTail(page)) ? !!(PageTail(page)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/mm.h", .line = 379, }; ______r = __builtin_expect(!!(PageTail(page)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/mm.h", .line = 379, }; ______r = !!(((__builtin_constant_p(PageTail(page)) ? !!(PageTail(page)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/mm.h", .line = 379, }; ______r = __builtin_expect(!!(PageTail(page)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) { do { (void)(atomic_read(&page->first_page->_count) <= 0); } while (0); atomic_inc(&page->first_page->_count); } } static inline __attribute__((always_inline)) struct page *virt_to_head_page(const void *x) { struct page *page = (mem_map + (((((unsigned long)(x)) - ((unsigned long)(0xC0000000UL))) >> 12) - (0UL))); return compound_head(page); } static inline __attribute__((always_inline)) void init_page_count(struct page *page) { atomic_set(&page->_count, 1); } static inline __attribute__((always_inline)) int PageBuddy(struct page *page) { return atomic_read(&page->_mapcount) == (-128); } static inline __attribute__((always_inline)) void __SetPageBuddy(struct page *page) { do { (void)(atomic_read(&page->_mapcount) != -1); } while (0); atomic_set(&page->_mapcount, (-128)); } static inline __attribute__((always_inline)) void __ClearPageBuddy(struct page *page) { do { (void)(!PageBuddy(page)); } while (0); atomic_set(&page->_mapcount, -1); } void put_page(struct page *page); void put_pages_list(struct list_head *pages); void split_page(struct page *page, unsigned int order); int split_free_page(struct page *page); typedef void compound_page_dtor(struct page *); static inline __attribute__((always_inline)) void set_compound_page_dtor(struct page *page, compound_page_dtor *dtor) { page[1].lru.next = (void *)dtor; } static inline __attribute__((always_inline)) compound_page_dtor *get_compound_page_dtor(struct page *page) { return (compound_page_dtor *)page[1].lru.next; } static inline __attribute__((always_inline)) int compound_order(struct page *page) { if (__builtin_constant_p(((!PageHead(page)))) ? !!((!PageHead(page))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/mm.h", .line = 459, }; ______r = !!((!PageHead(page))); ______f.miss_hit[______r]++; ______r; })) return 0; return (unsigned long)page[1].lru.prev; } static inline __attribute__((always_inline)) int compound_trans_order(struct page *page) { int order; unsigned long flags; if (__builtin_constant_p(((!PageHead(page)))) ? !!((!PageHead(page))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/mm.h", .line = 469, }; ______r = !!((!PageHead(page))); ______f.miss_hit[______r]++; ______r; })) return 0; flags = compound_lock_irqsave(page); order = compound_order(page); compound_unlock_irqrestore(page, flags); return order; } static inline __attribute__((always_inline)) void set_compound_order(struct page *page, unsigned long order) { page[1].lru.prev = (void *)order; } static inline __attribute__((always_inline)) pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma) { if (__builtin_constant_p((((__builtin_constant_p(vma->vm_flags & 0x00000002) ? !!(vma->vm_flags & 0x00000002) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/mm.h", .line = 492, }; ______r = __builtin_expect(!!(vma->vm_flags & 0x00000002), 1); ftrace_likely_update(&______f, ______r, 1); ______r; }))))) ? !!(((__builtin_constant_p(vma->vm_flags & 0x00000002) ? !!(vma->vm_flags & 0x00000002) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/mm.h", .line = 492, }; ______r = __builtin_expect(!!(vma->vm_flags & 0x00000002), 1); ftrace_likely_update(&______f, ______r, 1); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/mm.h", .line = 492, }; ______r = !!(((__builtin_constant_p(vma->vm_flags & 0x00000002) ? !!(vma->vm_flags & 0x00000002) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/mm.h", .line = 492, }; ______r = __builtin_expect(!!(vma->vm_flags & 0x00000002), 1); ftrace_likely_update(&______f, ______r, 1); ______r; })))); ______f.miss_hit[______r]++; ______r; })) pte = pte_mkwrite(pte); return pte; } static inline __attribute__((always_inline)) enum zone_type page_zonenum(struct page *page) { return (page->flags >> (((((sizeof(unsigned long)*8) - 0) - 0) - 2) * (2 != 0))) & ((1UL << 2) - 1); } static inline __attribute__((always_inline)) int page_zone_id(struct page *page) { return (page->flags >> ((((((sizeof(unsigned long)*8) - 0) - 0) < ((((sizeof(unsigned long)*8) - 0) - 0) - 2))? (((sizeof(unsigned long)*8) - 0) - 0) : ((((sizeof(unsigned long)*8) - 0) - 0) - 2)) * ((0 + 2) != 0))) & ((1UL << (0 + 2)) - 1); } static inline __attribute__((always_inline)) int zone_to_nid(struct zone *zone) { return 0; } static inline __attribute__((always_inline)) int page_to_nid(struct page *page) { return (page->flags >> ((((sizeof(unsigned long)*8) - 0) - 0) * (0 != 0))) & ((1UL << 0) - 1); } static inline __attribute__((always_inline)) struct zone *page_zone(struct page *page) { return &(&contig_page_data)->node_zones[page_zonenum(page)]; } static inline __attribute__((always_inline)) void set_page_zone(struct page *page, enum zone_type zone) { page->flags &= ~(((1UL << 2) - 1) << (((((sizeof(unsigned long)*8) - 0) - 0) - 2) * (2 != 0))); page->flags |= (zone & ((1UL << 2) - 1)) << (((((sizeof(unsigned long)*8) - 0) - 0) - 2) * (2 != 0)); } static inline __attribute__((always_inline)) void set_page_node(struct page *page, unsigned long node) { page->flags &= ~(((1UL << 0) - 1) << ((((sizeof(unsigned long)*8) - 0) - 0) * (0 != 0))); page->flags |= (node & ((1UL << 0) - 1)) << ((((sizeof(unsigned long)*8) - 0) - 0) * (0 != 0)); } static inline __attribute__((always_inline)) void set_page_links(struct page *page, enum zone_type zone, unsigned long node, unsigned long pfn) { set_page_zone(page, zone); set_page_node(page, node); } enum vm_event_item { PGPGIN, PGPGOUT, PSWPIN, PSWPOUT, PGALLOC_DMA, PGALLOC_NORMAL , PGALLOC_HIGH , PGALLOC_MOVABLE, PGFREE, PGACTIVATE, PGDEACTIVATE, PGFAULT, PGMAJFAULT, PGREFILL_DMA, PGREFILL_NORMAL , PGREFILL_HIGH , PGREFILL_MOVABLE, PGSTEAL_DMA, PGSTEAL_NORMAL , PGSTEAL_HIGH , PGSTEAL_MOVABLE, PGSCAN_KSWAPD_DMA, PGSCAN_KSWAPD_NORMAL , PGSCAN_KSWAPD_HIGH , PGSCAN_KSWAPD_MOVABLE, PGSCAN_DIRECT_DMA, PGSCAN_DIRECT_NORMAL , PGSCAN_DIRECT_HIGH , PGSCAN_DIRECT_MOVABLE, PGINODESTEAL, SLABS_SCANNED, KSWAPD_STEAL, KSWAPD_INODESTEAL, KSWAPD_LOW_WMARK_HIT_QUICKLY, KSWAPD_HIGH_WMARK_HIT_QUICKLY, KSWAPD_SKIP_CONGESTION_WAIT, PAGEOUTRUN, ALLOCSTALL, PGROTATED, COMPACTBLOCKS, COMPACTPAGES, COMPACTPAGEFAILED, COMPACTSTALL, COMPACTFAIL, COMPACTSUCCESS, UNEVICTABLE_PGCULLED, UNEVICTABLE_PGSCANNED, UNEVICTABLE_PGRESCUED, UNEVICTABLE_PGMLOCKED, UNEVICTABLE_PGMUNLOCKED, UNEVICTABLE_PGCLEARED, UNEVICTABLE_PGSTRANDED, UNEVICTABLE_MLOCKFREED, THP_FAULT_ALLOC, THP_FAULT_FALLBACK, THP_COLLAPSE_ALLOC, THP_COLLAPSE_ALLOC_FAILED, THP_SPLIT, NR_VM_EVENT_ITEMS }; extern int sysctl_stat_interval; struct vm_event_state { unsigned long event[NR_VM_EVENT_ITEMS]; }; extern __attribute__((section(".data..percpu" ""))) __typeof__(struct vm_event_state) vm_event_states; static inline __attribute__((always_inline)) void __count_vm_event(enum vm_event_item item) { do { do { const void *__vpp_verify = (typeof(&(((vm_event_states.event[item])))))((void *)0); (void)__vpp_verify; } while (0); switch(sizeof(((vm_event_states.event[item])))) { case 1: do { typedef typeof((((vm_event_states.event[item])))) pao_T__; const int pao_ID__ = (__builtin_constant_p((1)) && (((1)) == 1 || ((1)) == -1)) ? ((1)) : 0; if (__builtin_constant_p(((0))) ? !!((0)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 32, }; ______r = !!((0)); ______f.miss_hit[______r]++; ______r; })) { pao_T__ pao_tmp__; pao_tmp__ = ((1)); (void)pao_tmp__; } switch (sizeof((((vm_event_states.event[item]))))) { case 1: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 32, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incb ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 32, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decb ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else asm("addb %1, ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item])))) : "qi" ((pao_T__)((1)))); break; case 2: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 32, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incw ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 32, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decw ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else asm("addw %1, ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item])))) : "ri" ((pao_T__)((1)))); break; case 4: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 32, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incl ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 32, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decl ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else asm("addl %1, ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item])))) : "ri" ((pao_T__)((1)))); break; case 8: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 32, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incq ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 32, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decq ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else asm("addq %1, ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item])))) : "re" ((pao_T__)((1)))); break; default: __bad_percpu_size(); } } while (0);break; case 2: do { typedef typeof((((vm_event_states.event[item])))) pao_T__; const int pao_ID__ = (__builtin_constant_p((1)) && (((1)) == 1 || ((1)) == -1)) ? ((1)) : 0; if (__builtin_constant_p(((0))) ? !!((0)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 32, }; ______r = !!((0)); ______f.miss_hit[______r]++; ______r; })) { pao_T__ pao_tmp__; pao_tmp__ = ((1)); (void)pao_tmp__; } switch (sizeof((((vm_event_states.event[item]))))) { case 1: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 32, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incb ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 32, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decb ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else asm("addb %1, ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item])))) : "qi" ((pao_T__)((1)))); break; case 2: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 32, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incw ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 32, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decw ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else asm("addw %1, ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item])))) : "ri" ((pao_T__)((1)))); break; case 4: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 32, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incl ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 32, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decl ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else asm("addl %1, ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item])))) : "ri" ((pao_T__)((1)))); break; case 8: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 32, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incq ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 32, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decq ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else asm("addq %1, ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item])))) : "re" ((pao_T__)((1)))); break; default: __bad_percpu_size(); } } while (0);break; case 4: do { typedef typeof((((vm_event_states.event[item])))) pao_T__; const int pao_ID__ = (__builtin_constant_p((1)) && (((1)) == 1 || ((1)) == -1)) ? ((1)) : 0; if (__builtin_constant_p(((0))) ? !!((0)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 32, }; ______r = !!((0)); ______f.miss_hit[______r]++; ______r; })) { pao_T__ pao_tmp__; pao_tmp__ = ((1)); (void)pao_tmp__; } switch (sizeof((((vm_event_states.event[item]))))) { case 1: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 32, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incb ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 32, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decb ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else asm("addb %1, ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item])))) : "qi" ((pao_T__)((1)))); break; case 2: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 32, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incw ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 32, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decw ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else asm("addw %1, ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item])))) : "ri" ((pao_T__)((1)))); break; case 4: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 32, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incl ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 32, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decl ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else asm("addl %1, ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item])))) : "ri" ((pao_T__)((1)))); break; case 8: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 32, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incq ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 32, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decq ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else asm("addq %1, ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item])))) : "re" ((pao_T__)((1)))); break; default: __bad_percpu_size(); } } while (0);break; case 8: do { *({ unsigned long tcp_ptr__; do { const void *__vpp_verify = (typeof(&((((vm_event_states.event[item]))))))((void *)0); (void)__vpp_verify; } while (0); asm volatile("add " "%%""fs"":" "%P" "1" ", %0" : "=r" (tcp_ptr__) : "m" (this_cpu_off), "0" (&((((vm_event_states.event[item])))))); (typeof(*(&((((vm_event_states.event[item])))))) *)tcp_ptr__; }) += ((1)); } while (0);break; default: __bad_size_call_parameter();break; } } while (0); } static inline __attribute__((always_inline)) void count_vm_event(enum vm_event_item item) { do { do { const void *__vpp_verify = (typeof(&(((vm_event_states.event[item])))))((void *)0); (void)__vpp_verify; } while (0); switch(sizeof(((vm_event_states.event[item])))) { case 1: do { typedef typeof((((vm_event_states.event[item])))) pao_T__; const int pao_ID__ = (__builtin_constant_p((1)) && (((1)) == 1 || ((1)) == -1)) ? ((1)) : 0; if (__builtin_constant_p(((0))) ? !!((0)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 37, }; ______r = !!((0)); ______f.miss_hit[______r]++; ______r; })) { pao_T__ pao_tmp__; pao_tmp__ = ((1)); (void)pao_tmp__; } switch (sizeof((((vm_event_states.event[item]))))) { case 1: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 37, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incb ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 37, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decb ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else asm("addb %1, ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item])))) : "qi" ((pao_T__)((1)))); break; case 2: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 37, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incw ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 37, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decw ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else asm("addw %1, ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item])))) : "ri" ((pao_T__)((1)))); break; case 4: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 37, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incl ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 37, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decl ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else asm("addl %1, ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item])))) : "ri" ((pao_T__)((1)))); break; case 8: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 37, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incq ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 37, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decq ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else asm("addq %1, ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item])))) : "re" ((pao_T__)((1)))); break; default: __bad_percpu_size(); } } while (0);break; case 2: do { typedef typeof((((vm_event_states.event[item])))) pao_T__; const int pao_ID__ = (__builtin_constant_p((1)) && (((1)) == 1 || ((1)) == -1)) ? ((1)) : 0; if (__builtin_constant_p(((0))) ? !!((0)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 37, }; ______r = !!((0)); ______f.miss_hit[______r]++; ______r; })) { pao_T__ pao_tmp__; pao_tmp__ = ((1)); (void)pao_tmp__; } switch (sizeof((((vm_event_states.event[item]))))) { case 1: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 37, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incb ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 37, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decb ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else asm("addb %1, ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item])))) : "qi" ((pao_T__)((1)))); break; case 2: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 37, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incw ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 37, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decw ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else asm("addw %1, ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item])))) : "ri" ((pao_T__)((1)))); break; case 4: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 37, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incl ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 37, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decl ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else asm("addl %1, ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item])))) : "ri" ((pao_T__)((1)))); break; case 8: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 37, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incq ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 37, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decq ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else asm("addq %1, ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item])))) : "re" ((pao_T__)((1)))); break; default: __bad_percpu_size(); } } while (0);break; case 4: do { typedef typeof((((vm_event_states.event[item])))) pao_T__; const int pao_ID__ = (__builtin_constant_p((1)) && (((1)) == 1 || ((1)) == -1)) ? ((1)) : 0; if (__builtin_constant_p(((0))) ? !!((0)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 37, }; ______r = !!((0)); ______f.miss_hit[______r]++; ______r; })) { pao_T__ pao_tmp__; pao_tmp__ = ((1)); (void)pao_tmp__; } switch (sizeof((((vm_event_states.event[item]))))) { case 1: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 37, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incb ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 37, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decb ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else asm("addb %1, ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item])))) : "qi" ((pao_T__)((1)))); break; case 2: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 37, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incw ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 37, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decw ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else asm("addw %1, ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item])))) : "ri" ((pao_T__)((1)))); break; case 4: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 37, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incl ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 37, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decl ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else asm("addl %1, ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item])))) : "ri" ((pao_T__)((1)))); break; case 8: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 37, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incq ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 37, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decq ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item]))))); else asm("addq %1, ""%%""fs"":" "%P" "0" : "+m" ((((vm_event_states.event[item])))) : "re" ((pao_T__)((1)))); break; default: __bad_percpu_size(); } } while (0);break; case 8: do { do { add_preempt_count(1); __asm__ __volatile__("": : :"memory"); } while (0); *({ unsigned long tcp_ptr__; do { const void *__vpp_verify = (typeof(&((((vm_event_states.event[item]))))))((void *)0); (void)__vpp_verify; } while (0); asm volatile("add " "%%""fs"":" "%P" "1" ", %0" : "=r" (tcp_ptr__) : "m" (this_cpu_off), "0" (&((((vm_event_states.event[item])))))); (typeof(*(&((((vm_event_states.event[item])))))) *)tcp_ptr__; }) += ((1)); do { do { __asm__ __volatile__("": : :"memory"); sub_preempt_count(1); } while (0); __asm__ __volatile__("": : :"memory"); do { if (__builtin_constant_p((((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 37, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 37, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 37, }; ______r = !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 37, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) preempt_schedule(); } while (0); } while (0); } while (0);break; default: __bad_size_call_parameter();break; } } while (0); } static inline __attribute__((always_inline)) void __count_vm_events(enum vm_event_item item, long delta) { do { do { const void *__vpp_verify = (typeof(&((vm_event_states.event[item]))))((void *)0); (void)__vpp_verify; } while (0); switch(sizeof((vm_event_states.event[item]))) { case 1: do { typedef typeof(((vm_event_states.event[item]))) pao_T__; const int pao_ID__ = (__builtin_constant_p((delta)) && (((delta)) == 1 || ((delta)) == -1)) ? ((delta)) : 0; if (__builtin_constant_p(((0))) ? !!((0)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 42, }; ______r = !!((0)); ______f.miss_hit[______r]++; ______r; })) { pao_T__ pao_tmp__; pao_tmp__ = ((delta)); (void)pao_tmp__; } switch (sizeof(((vm_event_states.event[item])))) { case 1: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 42, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incb ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 42, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decb ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else asm("addb %1, ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item]))) : "qi" ((pao_T__)((delta)))); break; case 2: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 42, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incw ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 42, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decw ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else asm("addw %1, ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item]))) : "ri" ((pao_T__)((delta)))); break; case 4: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 42, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incl ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 42, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decl ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else asm("addl %1, ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item]))) : "ri" ((pao_T__)((delta)))); break; case 8: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 42, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incq ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 42, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decq ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else asm("addq %1, ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item]))) : "re" ((pao_T__)((delta)))); break; default: __bad_percpu_size(); } } while (0);break; case 2: do { typedef typeof(((vm_event_states.event[item]))) pao_T__; const int pao_ID__ = (__builtin_constant_p((delta)) && (((delta)) == 1 || ((delta)) == -1)) ? ((delta)) : 0; if (__builtin_constant_p(((0))) ? !!((0)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 42, }; ______r = !!((0)); ______f.miss_hit[______r]++; ______r; })) { pao_T__ pao_tmp__; pao_tmp__ = ((delta)); (void)pao_tmp__; } switch (sizeof(((vm_event_states.event[item])))) { case 1: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 42, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incb ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 42, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decb ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else asm("addb %1, ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item]))) : "qi" ((pao_T__)((delta)))); break; case 2: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 42, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incw ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 42, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decw ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else asm("addw %1, ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item]))) : "ri" ((pao_T__)((delta)))); break; case 4: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 42, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incl ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 42, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decl ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else asm("addl %1, ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item]))) : "ri" ((pao_T__)((delta)))); break; case 8: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 42, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incq ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 42, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decq ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else asm("addq %1, ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item]))) : "re" ((pao_T__)((delta)))); break; default: __bad_percpu_size(); } } while (0);break; case 4: do { typedef typeof(((vm_event_states.event[item]))) pao_T__; const int pao_ID__ = (__builtin_constant_p((delta)) && (((delta)) == 1 || ((delta)) == -1)) ? ((delta)) : 0; if (__builtin_constant_p(((0))) ? !!((0)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 42, }; ______r = !!((0)); ______f.miss_hit[______r]++; ______r; })) { pao_T__ pao_tmp__; pao_tmp__ = ((delta)); (void)pao_tmp__; } switch (sizeof(((vm_event_states.event[item])))) { case 1: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 42, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incb ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 42, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decb ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else asm("addb %1, ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item]))) : "qi" ((pao_T__)((delta)))); break; case 2: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 42, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incw ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 42, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decw ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else asm("addw %1, ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item]))) : "ri" ((pao_T__)((delta)))); break; case 4: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 42, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incl ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 42, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decl ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else asm("addl %1, ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item]))) : "ri" ((pao_T__)((delta)))); break; case 8: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 42, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incq ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 42, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decq ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else asm("addq %1, ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item]))) : "re" ((pao_T__)((delta)))); break; default: __bad_percpu_size(); } } while (0);break; case 8: do { *({ unsigned long tcp_ptr__; do { const void *__vpp_verify = (typeof(&(((vm_event_states.event[item])))))((void *)0); (void)__vpp_verify; } while (0); asm volatile("add " "%%""fs"":" "%P" "1" ", %0" : "=r" (tcp_ptr__) : "m" (this_cpu_off), "0" (&(((vm_event_states.event[item]))))); (typeof(*(&(((vm_event_states.event[item]))))) *)tcp_ptr__; }) += ((delta)); } while (0);break; default: __bad_size_call_parameter();break; } } while (0); } static inline __attribute__((always_inline)) void count_vm_events(enum vm_event_item item, long delta) { do { do { const void *__vpp_verify = (typeof(&((vm_event_states.event[item]))))((void *)0); (void)__vpp_verify; } while (0); switch(sizeof((vm_event_states.event[item]))) { case 1: do { typedef typeof(((vm_event_states.event[item]))) pao_T__; const int pao_ID__ = (__builtin_constant_p((delta)) && (((delta)) == 1 || ((delta)) == -1)) ? ((delta)) : 0; if (__builtin_constant_p(((0))) ? !!((0)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 47, }; ______r = !!((0)); ______f.miss_hit[______r]++; ______r; })) { pao_T__ pao_tmp__; pao_tmp__ = ((delta)); (void)pao_tmp__; } switch (sizeof(((vm_event_states.event[item])))) { case 1: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 47, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incb ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 47, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decb ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else asm("addb %1, ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item]))) : "qi" ((pao_T__)((delta)))); break; case 2: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 47, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incw ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 47, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decw ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else asm("addw %1, ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item]))) : "ri" ((pao_T__)((delta)))); break; case 4: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 47, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incl ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 47, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decl ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else asm("addl %1, ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item]))) : "ri" ((pao_T__)((delta)))); break; case 8: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 47, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incq ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 47, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decq ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else asm("addq %1, ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item]))) : "re" ((pao_T__)((delta)))); break; default: __bad_percpu_size(); } } while (0);break; case 2: do { typedef typeof(((vm_event_states.event[item]))) pao_T__; const int pao_ID__ = (__builtin_constant_p((delta)) && (((delta)) == 1 || ((delta)) == -1)) ? ((delta)) : 0; if (__builtin_constant_p(((0))) ? !!((0)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 47, }; ______r = !!((0)); ______f.miss_hit[______r]++; ______r; })) { pao_T__ pao_tmp__; pao_tmp__ = ((delta)); (void)pao_tmp__; } switch (sizeof(((vm_event_states.event[item])))) { case 1: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 47, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incb ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 47, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decb ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else asm("addb %1, ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item]))) : "qi" ((pao_T__)((delta)))); break; case 2: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 47, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incw ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 47, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decw ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else asm("addw %1, ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item]))) : "ri" ((pao_T__)((delta)))); break; case 4: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 47, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incl ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 47, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decl ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else asm("addl %1, ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item]))) : "ri" ((pao_T__)((delta)))); break; case 8: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 47, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incq ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 47, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decq ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else asm("addq %1, ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item]))) : "re" ((pao_T__)((delta)))); break; default: __bad_percpu_size(); } } while (0);break; case 4: do { typedef typeof(((vm_event_states.event[item]))) pao_T__; const int pao_ID__ = (__builtin_constant_p((delta)) && (((delta)) == 1 || ((delta)) == -1)) ? ((delta)) : 0; if (__builtin_constant_p(((0))) ? !!((0)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 47, }; ______r = !!((0)); ______f.miss_hit[______r]++; ______r; })) { pao_T__ pao_tmp__; pao_tmp__ = ((delta)); (void)pao_tmp__; } switch (sizeof(((vm_event_states.event[item])))) { case 1: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 47, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incb ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 47, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decb ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else asm("addb %1, ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item]))) : "qi" ((pao_T__)((delta)))); break; case 2: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 47, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incw ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 47, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decw ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else asm("addw %1, ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item]))) : "ri" ((pao_T__)((delta)))); break; case 4: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 47, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incl ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 47, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decl ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else asm("addl %1, ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item]))) : "ri" ((pao_T__)((delta)))); break; case 8: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 47, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incq ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 47, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decq ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item])))); else asm("addq %1, ""%%""fs"":" "%P" "0" : "+m" (((vm_event_states.event[item]))) : "re" ((pao_T__)((delta)))); break; default: __bad_percpu_size(); } } while (0);break; case 8: do { do { add_preempt_count(1); __asm__ __volatile__("": : :"memory"); } while (0); *({ unsigned long tcp_ptr__; do { const void *__vpp_verify = (typeof(&(((vm_event_states.event[item])))))((void *)0); (void)__vpp_verify; } while (0); asm volatile("add " "%%""fs"":" "%P" "1" ", %0" : "=r" (tcp_ptr__) : "m" (this_cpu_off), "0" (&(((vm_event_states.event[item]))))); (typeof(*(&(((vm_event_states.event[item]))))) *)tcp_ptr__; }) += ((delta)); do { do { __asm__ __volatile__("": : :"memory"); sub_preempt_count(1); } while (0); __asm__ __volatile__("": : :"memory"); do { if (__builtin_constant_p((((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 47, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 47, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 47, }; ______r = !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 47, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) preempt_schedule(); } while (0); } while (0); } while (0);break; default: __bad_size_call_parameter();break; } } while (0); } extern void all_vm_events(unsigned long *); extern void vm_events_fold_cpu(int cpu); extern atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS]; static inline __attribute__((always_inline)) void zone_page_state_add(long x, struct zone *zone, enum zone_stat_item item) { atomic_long_add(x, &zone->vm_stat[item]); atomic_long_add(x, &vm_stat[item]); } static inline __attribute__((always_inline)) unsigned long global_page_state(enum zone_stat_item item) { long x = atomic_long_read(&vm_stat[item]); if (__builtin_constant_p(((x < 0))) ? !!((x < 0)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 103, }; ______r = !!((x < 0)); ______f.miss_hit[______r]++; ______r; })) x = 0; return x; } static inline __attribute__((always_inline)) unsigned long zone_page_state(struct zone *zone, enum zone_stat_item item) { long x = atomic_long_read(&zone->vm_stat[item]); if (__builtin_constant_p(((x < 0))) ? !!((x < 0)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 114, }; ______r = !!((x < 0)); ______f.miss_hit[______r]++; ______r; })) x = 0; return x; } static inline __attribute__((always_inline)) unsigned long zone_page_state_snapshot(struct zone *zone, enum zone_stat_item item) { long x = atomic_long_read(&zone->vm_stat[item]); int cpu; for (((cpu)) = -1; ((cpu)) = cpumask_next(((cpu)), (cpu_online_mask)), ((cpu)) < nr_cpu_ids;) x += ({ do { const void *__vpp_verify = (typeof(((zone->pageset))))((void *)0); (void)__vpp_verify; } while (0); ({ unsigned long __ptr; __asm__ ("" : "=r"(__ptr) : "0"((typeof(*((zone->pageset))) *)((zone->pageset)))); (typeof((typeof(*((zone->pageset))) *)((zone->pageset)))) (__ptr + (((__per_cpu_offset[(cpu)])))); }); })->vm_stat_diff[item]; if (__builtin_constant_p(((x < 0))) ? !!((x < 0)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/vmstat.h", .line = 136, }; ______r = !!((x < 0)); ______f.miss_hit[______r]++; ______r; })) x = 0; return x; } extern unsigned long global_reclaimable_pages(void); extern unsigned long zone_reclaimable_pages(struct zone *zone); static inline __attribute__((always_inline)) void zap_zone_vm_stats(struct zone *zone) { __builtin_memset(zone->vm_stat, 0, sizeof(zone->vm_stat)); } extern void inc_zone_state(struct zone *, enum zone_stat_item); void __mod_zone_page_state(struct zone *, enum zone_stat_item item, int); void __inc_zone_page_state(struct page *, enum zone_stat_item); void __dec_zone_page_state(struct page *, enum zone_stat_item); void mod_zone_page_state(struct zone *, enum zone_stat_item, int); void inc_zone_page_state(struct page *, enum zone_stat_item); void dec_zone_page_state(struct page *, enum zone_stat_item); extern void inc_zone_state(struct zone *, enum zone_stat_item); extern void __inc_zone_state(struct zone *, enum zone_stat_item); extern void dec_zone_state(struct zone *, enum zone_stat_item); extern void __dec_zone_state(struct zone *, enum zone_stat_item); void refresh_cpu_vm_stats(int); void refresh_zone_stat_thresholds(void); int calculate_pressure_threshold(struct zone *zone); int calculate_normal_threshold(struct zone *zone); void set_pgdat_percpu_threshold(pg_data_t *pgdat, int (*calculate_pressure)(struct zone *)); extern const char * const vmstat_text[]; static inline __attribute__((always_inline)) __attribute__((always_inline)) void *lowmem_page_address(struct page *page) { return ((void *)((unsigned long)(((phys_addr_t)(((unsigned long)((page) - mem_map) + (0UL))) << 12))+((unsigned long)(0xC0000000UL)))); } void *page_address(struct page *page); void set_page_address(struct page *page, void *virtual); void page_address_init(void); extern struct address_space swapper_space; static inline __attribute__((always_inline)) struct address_space *page_mapping(struct page *page) { struct address_space *mapping = page->mapping; do { (void)(PageSlab(page)); } while (0); if (__builtin_constant_p((((__builtin_constant_p(PageSwapCache(page)) ? !!(PageSwapCache(page)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/mm.h", .line = 776, }; ______r = __builtin_expect(!!(PageSwapCache(page)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(PageSwapCache(page)) ? !!(PageSwapCache(page)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/mm.h", .line = 776, }; ______r = __builtin_expect(!!(PageSwapCache(page)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/mm.h", .line = 776, }; ______r = !!(((__builtin_constant_p(PageSwapCache(page)) ? !!(PageSwapCache(page)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/mm.h", .line = 776, }; ______r = __builtin_expect(!!(PageSwapCache(page)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) mapping = &swapper_space; else if (__builtin_constant_p((((unsigned long)mapping & 1))) ? !!(((unsigned long)mapping & 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/mm.h", .line = 778, }; ______r = !!(((unsigned long)mapping & 1)); ______f.miss_hit[______r]++; ______r; })) mapping = ((void *)0); return mapping; } static inline __attribute__((always_inline)) void *page_rmapping(struct page *page) { return (void *)((unsigned long)page->mapping & ~(1 | 2)); } static inline __attribute__((always_inline)) int PageAnon(struct page *page) { return ((unsigned long)page->mapping & 1) != 0; } static inline __attribute__((always_inline)) unsigned long page_index(struct page *page) { if (__builtin_constant_p((((__builtin_constant_p(PageSwapCache(page)) ? !!(PageSwapCache(page)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/mm.h", .line = 800, }; ______r = __builtin_expect(!!(PageSwapCache(page)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(PageSwapCache(page)) ? !!(PageSwapCache(page)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/mm.h", .line = 800, }; ______r = __builtin_expect(!!(PageSwapCache(page)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/mm.h", .line = 800, }; ______r = !!(((__builtin_constant_p(PageSwapCache(page)) ? !!(PageSwapCache(page)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/mm.h", .line = 800, }; ______r = __builtin_expect(!!(PageSwapCache(page)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) return ((page)->private); return page->index; } static inline __attribute__((always_inline)) void reset_page_mapcount(struct page *page) { atomic_set(&(page)->_mapcount, -1); } static inline __attribute__((always_inline)) int page_mapcount(struct page *page) { return atomic_read(&(page)->_mapcount) + 1; } static inline __attribute__((always_inline)) int page_mapped(struct page *page) { return atomic_read(&(page)->_mapcount) >= 0; } extern void pagefault_out_of_memory(void); extern void show_free_areas(unsigned int flags); extern bool skip_free_areas_node(unsigned int flags, int nid); int shmem_lock(struct file *file, int lock, struct user_struct *user); struct file *shmem_file_setup(const char *name, loff_t size, unsigned long flags); int shmem_zero_setup(struct vm_area_struct *); extern int can_do_mlock(void); extern int user_shm_lock(size_t, struct user_struct *); extern void user_shm_unlock(size_t, struct user_struct *); struct zap_details { struct vm_area_struct *nonlinear_vma; struct address_space *check_mapping; unsigned long first_index; unsigned long last_index; }; struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr, pte_t pte); int zap_vma_ptes(struct vm_area_struct *vma, unsigned long address, unsigned long size); unsigned long zap_page_range(struct vm_area_struct *vma, unsigned long address, unsigned long size, struct zap_details *); unsigned long unmap_vmas(struct mmu_gather *tlb, struct vm_area_struct *start_vma, unsigned long start_addr, unsigned long end_addr, unsigned long *nr_accounted, struct zap_details *); struct mm_walk { int (*pgd_entry)(pgd_t *, unsigned long, unsigned long, struct mm_walk *); int (*pud_entry)(pud_t *, unsigned long, unsigned long, struct mm_walk *); int (*pmd_entry)(pmd_t *, unsigned long, unsigned long, struct mm_walk *); int (*pte_entry)(pte_t *, unsigned long, unsigned long, struct mm_walk *); int (*pte_hole)(unsigned long, unsigned long, struct mm_walk *); int (*hugetlb_entry)(pte_t *, unsigned long, unsigned long, unsigned long, struct mm_walk *); struct mm_struct *mm; void *private; }; int walk_page_range(unsigned long addr, unsigned long end, struct mm_walk *walk); void free_pgd_range(struct mmu_gather *tlb, unsigned long addr, unsigned long end, unsigned long floor, unsigned long ceiling); int copy_page_range(struct mm_struct *dst, struct mm_struct *src, struct vm_area_struct *vma); void unmap_mapping_range(struct address_space *mapping, loff_t const holebegin, loff_t const holelen, int even_cows); int follow_pfn(struct vm_area_struct *vma, unsigned long address, unsigned long *pfn); int follow_phys(struct vm_area_struct *vma, unsigned long address, unsigned int flags, unsigned long *prot, resource_size_t *phys); int generic_access_phys(struct vm_area_struct *vma, unsigned long addr, void *buf, int len, int write); static inline __attribute__((always_inline)) void unmap_shared_mapping_range(struct address_space *mapping, loff_t const holebegin, loff_t const holelen) { unmap_mapping_range(mapping, holebegin, holelen, 0); } extern void truncate_pagecache(struct inode *inode, loff_t old, loff_t new); extern void truncate_setsize(struct inode *inode, loff_t newsize); extern int vmtruncate(struct inode *inode, loff_t offset); extern int vmtruncate_range(struct inode *inode, loff_t offset, loff_t end); int truncate_inode_page(struct address_space *mapping, struct page *page); int generic_error_remove_page(struct address_space *mapping, struct page *page); int invalidate_inode_page(struct page *page); extern int handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long address, unsigned int flags); extern int fixup_user_fault(struct task_struct *tsk, struct mm_struct *mm, unsigned long address, unsigned int fault_flags); extern int make_pages_present(unsigned long addr, unsigned long end); extern int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write); extern int access_remote_vm(struct mm_struct *mm, unsigned long addr, void *buf, int len, int write); int __get_user_pages(struct task_struct *tsk, struct mm_struct *mm, unsigned long start, int len, unsigned int foll_flags, struct page **pages, struct vm_area_struct **vmas, int *nonblocking); int get_user_pages(struct task_struct *tsk, struct mm_struct *mm, unsigned long start, int nr_pages, int write, int force, struct page **pages, struct vm_area_struct **vmas); int get_user_pages_fast(unsigned long start, int nr_pages, int write, struct page **pages); struct page *get_dump_page(unsigned long addr); extern int try_to_release_page(struct page * page, gfp_t gfp_mask); extern void do_invalidatepage(struct page *page, unsigned long offset); int __set_page_dirty_nobuffers(struct page *page); int __set_page_dirty_no_writeback(struct page *page); int redirty_page_for_writepage(struct writeback_control *wbc, struct page *page); void account_page_dirtied(struct page *page, struct address_space *mapping); void account_page_writeback(struct page *page); int set_page_dirty(struct page *page); int set_page_dirty_lock(struct page *page); int clear_page_dirty_for_io(struct page *page); static inline __attribute__((always_inline)) int vma_growsdown(struct vm_area_struct *vma, unsigned long addr) { return vma && (vma->vm_end == addr) && (vma->vm_flags & 0x00000100); } static inline __attribute__((always_inline)) int stack_guard_page_start(struct vm_area_struct *vma, unsigned long addr) { return (vma->vm_flags & 0x00000100) && (vma->vm_start == addr) && !vma_growsdown(vma->vm_prev, addr); } static inline __attribute__((always_inline)) int vma_growsup(struct vm_area_struct *vma, unsigned long addr) { return vma && (vma->vm_start == addr) && (vma->vm_flags & 0x00000000); } static inline __attribute__((always_inline)) int stack_guard_page_end(struct vm_area_struct *vma, unsigned long addr) { return (vma->vm_flags & 0x00000000) && (vma->vm_end == addr) && !vma_growsup(vma->vm_next, addr); } extern unsigned long move_page_tables(struct vm_area_struct *vma, unsigned long old_addr, struct vm_area_struct *new_vma, unsigned long new_addr, unsigned long len); extern unsigned long do_mremap(unsigned long addr, unsigned long old_len, unsigned long new_len, unsigned long flags, unsigned long new_addr); extern int mprotect_fixup(struct vm_area_struct *vma, struct vm_area_struct **pprev, unsigned long start, unsigned long end, unsigned long newflags); int __get_user_pages_fast(unsigned long start, int nr_pages, int write, struct page **pages); static inline __attribute__((always_inline)) void set_mm_counter(struct mm_struct *mm, int member, long value) { atomic_long_set(&mm->rss_stat.count[member], value); } static inline __attribute__((always_inline)) unsigned long get_mm_counter(struct mm_struct *mm, int member) { return atomic_long_read(&mm->rss_stat.count[member]); } static inline __attribute__((always_inline)) void add_mm_counter(struct mm_struct *mm, int member, long value) { atomic_long_add(value, &mm->rss_stat.count[member]); } static inline __attribute__((always_inline)) void inc_mm_counter(struct mm_struct *mm, int member) { atomic_long_inc(&mm->rss_stat.count[member]); } static inline __attribute__((always_inline)) void dec_mm_counter(struct mm_struct *mm, int member) { atomic_long_dec(&mm->rss_stat.count[member]); } static inline __attribute__((always_inline)) unsigned long get_mm_rss(struct mm_struct *mm) { return get_mm_counter(mm, MM_FILEPAGES) + get_mm_counter(mm, MM_ANONPAGES); } static inline __attribute__((always_inline)) unsigned long get_mm_hiwater_rss(struct mm_struct *mm) { return ({ typeof(mm->hiwater_rss) _max1 = (mm->hiwater_rss); typeof(get_mm_rss(mm)) _max2 = (get_mm_rss(mm)); (void) (&_max1 == &_max2); _max1 > _max2 ? _max1 : _max2; }); } static inline __attribute__((always_inline)) unsigned long get_mm_hiwater_vm(struct mm_struct *mm) { return ({ typeof(mm->hiwater_vm) _max1 = (mm->hiwater_vm); typeof(mm->total_vm) _max2 = (mm->total_vm); (void) (&_max1 == &_max2); _max1 > _max2 ? _max1 : _max2; }); } static inline __attribute__((always_inline)) void update_hiwater_rss(struct mm_struct *mm) { unsigned long _rss = get_mm_rss(mm); if (__builtin_constant_p((((mm)->hiwater_rss < _rss))) ? !!(((mm)->hiwater_rss < _rss)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/mm.h", .line = 1107, }; ______r = !!(((mm)->hiwater_rss < _rss)); ______f.miss_hit[______r]++; ______r; })) (mm)->hiwater_rss = _rss; } static inline __attribute__((always_inline)) void update_hiwater_vm(struct mm_struct *mm) { if (__builtin_constant_p(((mm->hiwater_vm < mm->total_vm))) ? !!((mm->hiwater_vm < mm->total_vm)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/mm.h", .line = 1113, }; ______r = !!((mm->hiwater_vm < mm->total_vm)); ______f.miss_hit[______r]++; ______r; })) mm->hiwater_vm = mm->total_vm; } static inline __attribute__((always_inline)) void setmax_mm_hiwater_rss(unsigned long *maxrss, struct mm_struct *mm) { unsigned long hiwater_rss = get_mm_hiwater_rss(mm); if (__builtin_constant_p(((*maxrss < hiwater_rss))) ? !!((*maxrss < hiwater_rss)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/mm.h", .line = 1122, }; ______r = !!((*maxrss < hiwater_rss)); ______f.miss_hit[______r]++; ______r; })) *maxrss = hiwater_rss; } static inline __attribute__((always_inline)) void sync_mm_rss(struct task_struct *task, struct mm_struct *mm) { } struct shrink_control { gfp_t gfp_mask; unsigned long nr_to_scan; }; struct shrinker { int (*shrink)(struct shrinker *, struct shrink_control *sc); int seeks; struct list_head list; long nr; }; extern void register_shrinker(struct shrinker *); extern void unregister_shrinker(struct shrinker *); int vma_wants_writenotify(struct vm_area_struct *vma); extern pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr, spinlock_t **ptl); static inline __attribute__((always_inline)) pte_t *get_locked_pte(struct mm_struct *mm, unsigned long addr, spinlock_t **ptl) { pte_t *ptep; (ptep = __get_locked_pte(mm, addr, ptl)); return ptep; } static inline __attribute__((always_inline)) int __pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address) { return 0; } int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address); int __pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma, pmd_t *pmd, unsigned long address); int __pte_alloc_kernel(pmd_t *pmd, unsigned long address); static inline __attribute__((always_inline)) pud_t *pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address) { return ((__builtin_constant_p(pgd_none(*pgd)) ? !!(pgd_none(*pgd)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/mm.h", .line = 1215, }; ______r = __builtin_expect(!!(pgd_none(*pgd)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })) && __pud_alloc(mm, pgd, address))? ((void *)0): pud_offset(pgd, address); } static inline __attribute__((always_inline)) pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address) { return ((__builtin_constant_p(pud_none(*pud)) ? !!(pud_none(*pud)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/mm.h", .line = 1221, }; ______r = __builtin_expect(!!(pud_none(*pud)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })) && __pmd_alloc(mm, pud, address))? ((void *)0): pmd_offset(pud, address); } static inline __attribute__((always_inline)) void pgtable_page_ctor(struct page *page) { do {} while (0); inc_zone_page_state(page, NR_PAGETABLE); } static inline __attribute__((always_inline)) void pgtable_page_dtor(struct page *page) { do {} while (0); dec_zone_page_state(page, NR_PAGETABLE); } extern void free_area_init(unsigned long * zones_size); extern void free_area_init_node(int nid, unsigned long * zones_size, unsigned long zone_start_pfn, unsigned long *zholes_size); extern void free_area_init_nodes(unsigned long *max_zone_pfn); extern void add_active_range(unsigned int nid, unsigned long start_pfn, unsigned long end_pfn); extern void remove_active_range(unsigned int nid, unsigned long start_pfn, unsigned long end_pfn); extern void remove_all_active_ranges(void); void sort_node_map(void); unsigned long __absent_pages_in_range(int nid, unsigned long start_pfn, unsigned long end_pfn); extern unsigned long absent_pages_in_range(unsigned long start_pfn, unsigned long end_pfn); extern void get_pfn_range_for_nid(unsigned int nid, unsigned long *start_pfn, unsigned long *end_pfn); extern unsigned long find_min_pfn_with_active_regions(void); extern void free_bootmem_with_active_regions(int nid, unsigned long max_low_pfn); int add_from_early_node_map(struct range *range, int az, int nr_range, int nid); u64 __attribute__ ((__section__(".init.text"))) __attribute__((__cold__)) __attribute__((no_instrument_function)) find_memory_core_early(int nid, u64 size, u64 align, u64 goal, u64 limit); typedef int (*work_fn_t)(unsigned long, unsigned long, void *); extern void work_with_active_regions(int nid, work_fn_t work_fn, void *data); extern void sparse_memory_present_with_active_regions(int nid); extern int __attribute__ ((__section__(".meminit.text"))) __attribute__((__cold__)) __attribute__((no_instrument_function)) early_pfn_to_nid(unsigned long pfn); extern void set_dma_reserve(unsigned long new_dma_reserve); extern void memmap_init_zone(unsigned long, int, unsigned long, unsigned long, enum memmap_context); extern void setup_per_zone_wmarks(void); extern int __attribute__ ((__section__(".meminit.text"))) __attribute__((__cold__)) __attribute__((no_instrument_function)) init_per_zone_wmark_min(void); extern void mem_init(void); extern void __attribute__ ((__section__(".init.text"))) __attribute__((__cold__)) __attribute__((no_instrument_function)) mmap_init(void); extern void show_mem(unsigned int flags); extern void si_meminfo(struct sysinfo * val); extern void si_meminfo_node(struct sysinfo *val, int nid); extern int after_bootmem; extern void warn_alloc_failed(gfp_t gfp_mask, int order, const char *fmt, ...); extern void setup_per_cpu_pageset(void); extern void zone_pcp_update(struct zone *zone); extern atomic_long_t mmap_pages_allocated; extern int nommu_shrink_inode_mappings(struct inode *, size_t, size_t); void vma_prio_tree_add(struct vm_area_struct *, struct vm_area_struct *old); void vma_prio_tree_insert(struct vm_area_struct *, struct prio_tree_root *); void vma_prio_tree_remove(struct vm_area_struct *, struct prio_tree_root *); struct vm_area_struct *vma_prio_tree_next(struct vm_area_struct *vma, struct prio_tree_iter *iter); static inline __attribute__((always_inline)) void vma_nonlinear_insert(struct vm_area_struct *vma, struct list_head *list) { vma->shared.vm_set.parent = ((void *)0); list_add_tail(&vma->shared.vm_set.list, list); } extern int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin); extern int vma_adjust(struct vm_area_struct *vma, unsigned long start, unsigned long end, unsigned long pgoff, struct vm_area_struct *insert); extern struct vm_area_struct *vma_merge(struct mm_struct *, struct vm_area_struct *prev, unsigned long addr, unsigned long end, unsigned long vm_flags, struct anon_vma *, struct file *, unsigned long, struct mempolicy *); extern struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *); extern int split_vma(struct mm_struct *, struct vm_area_struct *, unsigned long addr, int new_below); extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *); extern void __vma_link_rb(struct mm_struct *, struct vm_area_struct *, struct rb_node **, struct rb_node *); extern void unlink_file_vma(struct vm_area_struct *); extern struct vm_area_struct *copy_vma(struct vm_area_struct **, unsigned long addr, unsigned long len, unsigned long pgoff); extern void exit_mmap(struct mm_struct *); extern int mm_take_all_locks(struct mm_struct *mm); extern void mm_drop_all_locks(struct mm_struct *mm); extern void added_exe_file_vma(struct mm_struct *mm); extern void removed_exe_file_vma(struct mm_struct *mm); extern void set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file); extern struct file *get_mm_exe_file(struct mm_struct *mm); extern int may_expand_vm(struct mm_struct *mm, unsigned long npages); extern int install_special_mapping(struct mm_struct *mm, unsigned long addr, unsigned long len, unsigned long flags, struct page **pages); extern unsigned long get_unmapped_area(struct file *, unsigned long, unsigned long, unsigned long, unsigned long); extern unsigned long do_mmap_pgoff(struct file *file, unsigned long addr, unsigned long len, unsigned long prot, unsigned long flag, unsigned long pgoff); extern unsigned long mmap_region(struct file *file, unsigned long addr, unsigned long len, unsigned long flags, vm_flags_t vm_flags, unsigned long pgoff); static inline __attribute__((always_inline)) unsigned long do_mmap(struct file *file, unsigned long addr, unsigned long len, unsigned long prot, unsigned long flag, unsigned long offset) { unsigned long ret = -22; if (__builtin_constant_p((((offset + ((((len)) + ((typeof((len)))((((1UL) << 12))) - 1)) & ~((typeof((len)))((((1UL) << 12))) - 1))) < offset))) ? !!(((offset + ((((len)) + ((typeof((len)))((((1UL) << 12))) - 1)) & ~((typeof((len)))((((1UL) << 12))) - 1))) < offset)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/mm.h", .line = 1446, }; ______r = !!(((offset + ((((len)) + ((typeof((len)))((((1UL) << 12))) - 1)) & ~((typeof((len)))((((1UL) << 12))) - 1))) < offset)); ______f.miss_hit[______r]++; ______r; })) goto out; if (__builtin_constant_p(((!(offset & ~(~(((1UL) << 12)-1)))))) ? !!((!(offset & ~(~(((1UL) << 12)-1))))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/mm.h", .line = 1448, }; ______r = !!((!(offset & ~(~(((1UL) << 12)-1))))); ______f.miss_hit[______r]++; ______r; })) ret = do_mmap_pgoff(file, addr, len, prot, flag, offset >> 12); out: return ret; } extern int do_munmap(struct mm_struct *, unsigned long, size_t); extern unsigned long do_brk(unsigned long, unsigned long); extern unsigned long page_unuse(struct page *); extern void truncate_inode_pages(struct address_space *, loff_t); extern void truncate_inode_pages_range(struct address_space *, loff_t lstart, loff_t lend); extern int filemap_fault(struct vm_area_struct *, struct vm_fault *); int write_one_page(struct page *page, int wait); void task_dirty_inc(struct task_struct *tsk); int force_page_cache_readahead(struct address_space *mapping, struct file *filp, unsigned long offset, unsigned long nr_to_read); void page_cache_sync_readahead(struct address_space *mapping, struct file_ra_state *ra, struct file *filp, unsigned long offset, unsigned long size); void page_cache_async_readahead(struct address_space *mapping, struct file_ra_state *ra, struct file *filp, struct page *pg, unsigned long offset, unsigned long size); unsigned long max_sane_readahead(unsigned long nr); unsigned long ra_submit(struct file_ra_state *ra, struct address_space *mapping, struct file *filp); extern int expand_stack(struct vm_area_struct *vma, unsigned long address); extern int expand_downwards(struct vm_area_struct *vma, unsigned long address); extern struct vm_area_struct * find_vma(struct mm_struct * mm, unsigned long addr); extern struct vm_area_struct * find_vma_prev(struct mm_struct * mm, unsigned long addr, struct vm_area_struct **pprev); static inline __attribute__((always_inline)) struct vm_area_struct * find_vma_intersection(struct mm_struct * mm, unsigned long start_addr, unsigned long end_addr) { struct vm_area_struct * vma = find_vma(mm,start_addr); if (__builtin_constant_p(((vma && end_addr <= vma->vm_start))) ? !!((vma && end_addr <= vma->vm_start)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/mm.h", .line = 1519, }; ______r = !!((vma && end_addr <= vma->vm_start)); ______f.miss_hit[______r]++; ______r; })) vma = ((void *)0); return vma; } static inline __attribute__((always_inline)) unsigned long vma_pages(struct vm_area_struct *vma) { return (vma->vm_end - vma->vm_start) >> 12; } pgprot_t vm_get_page_prot(unsigned long vm_flags); struct vm_area_struct *find_extend_vma(struct mm_struct *, unsigned long addr); int remap_pfn_range(struct vm_area_struct *, unsigned long addr, unsigned long pfn, unsigned long size, pgprot_t); int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *); int vm_insert_pfn(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn); int vm_insert_mixed(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn); struct page *follow_page(struct vm_area_struct *, unsigned long address, unsigned int foll_flags); typedef int (*pte_fn_t)(pte_t *pte, pgtable_t token, unsigned long addr, void *data); extern int apply_to_page_range(struct mm_struct *mm, unsigned long address, unsigned long size, pte_fn_t fn, void *data); void vm_stat_account(struct mm_struct *, unsigned long, struct file *, long); static inline __attribute__((always_inline)) void kernel_map_pages(struct page *page, int numpages, int enable) {} static inline __attribute__((always_inline)) void enable_debug_pagealloc(void) { } static inline __attribute__((always_inline)) bool kernel_page_present(struct page *page) { return true; } extern struct vm_area_struct *get_gate_vma(struct mm_struct *mm); int in_gate_area_no_mm(unsigned long addr); int in_gate_area(struct mm_struct *mm, unsigned long addr); int drop_caches_sysctl_handler(struct ctl_table *, int, void *, size_t *, loff_t *); unsigned long shrink_slab(struct shrink_control *shrink, unsigned long nr_pages_scanned, unsigned long lru_pages); extern int randomize_va_space; const char * arch_vma_name(struct vm_area_struct *vma); void print_vma_addr(char *prefix, unsigned long rip); void sparse_mem_maps_populate_node(struct page **map_map, unsigned long pnum_begin, unsigned long pnum_end, unsigned long map_count, int nodeid); struct page *sparse_mem_map_populate(unsigned long pnum, int nid); pgd_t *vmemmap_pgd_populate(unsigned long addr, int node); pud_t *vmemmap_pud_populate(pgd_t *pgd, unsigned long addr, int node); pmd_t *vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node); pte_t *vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node); void *vmemmap_alloc_block(unsigned long size, int node); void *vmemmap_alloc_block_buf(unsigned long size, int node); void vmemmap_verify(pte_t *, int, unsigned long, unsigned long); int vmemmap_populate_basepages(struct page *start_page, unsigned long pages, int node); int vmemmap_populate(struct page *start_page, unsigned long pages, int node); void vmemmap_populate_print_last(void); enum mf_flags { MF_COUNT_INCREASED = 1 << 0, }; extern void memory_failure(unsigned long pfn, int trapno); extern int __memory_failure(unsigned long pfn, int trapno, int flags); extern int unpoison_memory(unsigned long pfn); extern int sysctl_memory_failure_early_kill; extern int sysctl_memory_failure_recovery; extern void shake_page(struct page *p, int access); extern atomic_long_t mce_bad_pages; extern int soft_offline_page(struct page *page, int flags); extern void dump_page(struct page *page); extern void clear_huge_page(struct page *page, unsigned long addr, unsigned int pages_per_huge_page); extern void copy_user_huge_page(struct page *dst, struct page *src, unsigned long addr, struct vm_area_struct *vma, unsigned int pages_per_huge_page); static inline __attribute__((always_inline)) unsigned long get_page_memtype(struct page *pg) { unsigned long pg_flags = pg->flags & (1UL << PG_uncached | 1UL << PG_arch_1); if (__builtin_constant_p(((pg_flags == 0))) ? !!((pg_flags == 0)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/cacheflush.h", .line = 28, }; ______r = !!((pg_flags == 0)); ______f.miss_hit[______r]++; ______r; })) return -1; else if (__builtin_constant_p(((pg_flags == (1UL << PG_arch_1)))) ? !!((pg_flags == (1UL << PG_arch_1))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/cacheflush.h", .line = 30, }; ______r = !!((pg_flags == (1UL << PG_arch_1))); ______f.miss_hit[______r]++; ______r; })) return ((((pteval_t)(1)) << 3)); else if (__builtin_constant_p(((pg_flags == (1UL << PG_uncached)))) ? !!((pg_flags == (1UL << PG_uncached))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/cacheflush.h", .line = 32, }; ______r = !!((pg_flags == (1UL << PG_uncached))); ______f.miss_hit[______r]++; ______r; })) return ((((pteval_t)(1)) << 4)); else return (0); } static inline __attribute__((always_inline)) void set_page_memtype(struct page *pg, unsigned long memtype) { unsigned long memtype_flags = 0; unsigned long old_flags; unsigned long new_flags; switch (memtype) { case ((((pteval_t)(1)) << 3)): memtype_flags = (1UL << PG_arch_1); break; case ((((pteval_t)(1)) << 4)): memtype_flags = (1UL << PG_uncached); break; case (0): memtype_flags = (1UL << PG_uncached | 1UL << PG_arch_1); break; } do { old_flags = pg->flags; new_flags = (old_flags & (~(1UL << PG_uncached | 1UL << PG_arch_1))) | memtype_flags; } while (({ __typeof__(*(((&pg->flags)))) __ret; __typeof__(*(((&pg->flags)))) __old = (((old_flags))); __typeof__(*(((&pg->flags)))) __new = (((new_flags))); switch ((sizeof(*&pg->flags))) { case 1: { volatile u8 *__ptr = (volatile u8 *)(((&pg->flags))); asm volatile(".section .smp_locks,\"a\"\n" ".balign 4\n" ".long 671f - .\n" ".previous\n" "671:" "\n\tlock; " "cmpxchgb %2,%1" : "=a" (__ret), "+m" (*__ptr) : "q" (__new), "0" (__old) : "memory"); break; } case 2: { volatile u16 *__ptr = (volatile u16 *)(((&pg->flags))); asm volatile(".section .smp_locks,\"a\"\n" ".balign 4\n" ".long 671f - .\n" ".previous\n" "671:" "\n\tlock; " "cmpxchgw %2,%1" : "=a" (__ret), "+m" (*__ptr) : "r" (__new), "0" (__old) : "memory"); break; } case 4: { volatile u32 *__ptr = (volatile u32 *)(((&pg->flags))); asm volatile(".section .smp_locks,\"a\"\n" ".balign 4\n" ".long 671f - .\n" ".previous\n" "671:" "\n\tlock; " "cmpxchgl %2,%1" : "=a" (__ret), "+m" (*__ptr) : "r" (__new), "0" (__old) : "memory"); break; } default: __cmpxchg_wrong_size(); } __ret; }) != old_flags); } int _set_memory_uc(unsigned long addr, int numpages); int _set_memory_wc(unsigned long addr, int numpages); int _set_memory_wb(unsigned long addr, int numpages); int set_memory_uc(unsigned long addr, int numpages); int set_memory_wc(unsigned long addr, int numpages); int set_memory_wb(unsigned long addr, int numpages); int set_memory_x(unsigned long addr, int numpages); int set_memory_nx(unsigned long addr, int numpages); int set_memory_ro(unsigned long addr, int numpages); int set_memory_rw(unsigned long addr, int numpages); int set_memory_np(unsigned long addr, int numpages); int set_memory_4k(unsigned long addr, int numpages); int set_memory_array_uc(unsigned long *addr, int addrinarray); int set_memory_array_wc(unsigned long *addr, int addrinarray); int set_memory_array_wb(unsigned long *addr, int addrinarray); int set_pages_array_uc(struct page **pages, int addrinarray); int set_pages_array_wc(struct page **pages, int addrinarray); int set_pages_array_wb(struct page **pages, int addrinarray); int set_pages_uc(struct page *page, int numpages); int set_pages_wb(struct page *page, int numpages); int set_pages_x(struct page *page, int numpages); int set_pages_nx(struct page *page, int numpages); int set_pages_ro(struct page *page, int numpages); int set_pages_rw(struct page *page, int numpages); void clflush_cache_range(void *addr, unsigned int size); void mark_rodata_ro(void); extern const int rodata_test_data; extern int kernel_set_to_readonly; void set_kernel_text_rw(void); void set_kernel_text_ro(void); static inline __attribute__((always_inline)) int rodata_test(void) { return 0; } static inline __attribute__((always_inline)) void flush_anon_page(struct vm_area_struct *vma, struct page *page, unsigned long vmaddr) { } static inline __attribute__((always_inline)) void flush_kernel_dcache_page(struct page *page) { } static inline __attribute__((always_inline)) void flush_kernel_vmap_range(void *vaddr, int size) { } static inline __attribute__((always_inline)) void invalidate_kernel_vmap_range(void *vaddr, int size) { } static inline __attribute__((always_inline)) void __native_flush_tlb(void) { native_write_cr3(native_read_cr3()); } static inline __attribute__((always_inline)) void __native_flush_tlb_global(void) { unsigned long flags; unsigned long cr4; do { ({ unsigned long __dummy; typeof(flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); flags = arch_local_irq_save(); } while (0); cr4 = native_read_cr4(); native_write_cr4(cr4 & ~0x00000080); native_write_cr4(cr4); do { ({ unsigned long __dummy; typeof(flags) __dummy2; (void)(&__dummy == &__dummy2); 1; }); arch_local_irq_restore(flags); } while (0); } static inline __attribute__((always_inline)) void __native_flush_tlb_single(unsigned long addr) { asm volatile("invlpg (%0)" ::"r" (addr) : "memory"); } static inline __attribute__((always_inline)) void __flush_tlb_all(void) { if (__builtin_constant_p((((__builtin_constant_p((0*32+13)) && ( ((((0*32+13))>>5)==0 && (1UL<<(((0*32+13))&31) & ((1<<((0*32+ 0) & 31))|0|0|(1<<((0*32+ 6) & 31))| (1<<((0*32+ 8) & 31))|0|0|(1<<((0*32+15) & 31))| 0|0))) || ((((0*32+13))>>5)==1 && (1UL<<(((0*32+13))&31) & (0|0))) || ((((0*32+13))>>5)==2 && (1UL<<(((0*32+13))&31) & 0)) || ((((0*32+13))>>5)==3 && (1UL<<(((0*32+13))&31) & (0))) || ((((0*32+13))>>5)==4 && (1UL<<(((0*32+13))&31) & 0)) || ((((0*32+13))>>5)==5 && (1UL<<(((0*32+13))&31) & 0)) || ((((0*32+13))>>5)==6 && (1UL<<(((0*32+13))&31) & 0)) || ((((0*32+13))>>5)==7 && (1UL<<(((0*32+13))&31) & 0)) || ((((0*32+13))>>5)==8 && (1UL<<(((0*32+13))&31) & 0)) || ((((0*32+13))>>5)==9 && (1UL<<(((0*32+13))&31) & 0)) ) ? 1 : (__builtin_constant_p(((0*32+13))) ? constant_test_bit(((0*32+13)), ((unsigned long *)((&boot_cpu_data)->x86_capability))) : variable_test_bit(((0*32+13)), ((unsigned long *)((&boot_cpu_data)->x86_capability)))))))) ? !!(((__builtin_constant_p((0*32+13)) && ( ((((0*32+13))>>5)==0 && (1UL<<(((0*32+13))&31) & ((1<<((0*32+ 0) & 31))|0|0|(1<<((0*32+ 6) & 31))| (1<<((0*32+ 8) & 31))|0|0|(1<<((0*32+15) & 31))| 0|0))) || ((((0*32+13))>>5)==1 && (1UL<<(((0*32+13))&31) & (0|0))) || ((((0*32+13))>>5)==2 && (1UL<<(((0*32+13))&31) & 0)) || ((((0*32+13))>>5)==3 && (1UL<<(((0*32+13))&31) & (0))) || ((((0*32+13))>>5)==4 && (1UL<<(((0*32+13))&31) & 0)) || ((((0*32+13))>>5)==5 && (1UL<<(((0*32+13))&31) & 0)) || ((((0*32+13))>>5)==6 && (1UL<<(((0*32+13))&31) & 0)) || ((((0*32+13))>>5)==7 && (1UL<<(((0*32+13))&31) & 0)) || ((((0*32+13))>>5)==8 && (1UL<<(((0*32+13))&31) & 0)) || ((((0*32+13))>>5)==9 && (1UL<<(((0*32+13))&31) & 0)) ) ? 1 : (__builtin_constant_p(((0*32+13))) ? constant_test_bit(((0*32+13)), ((unsigned long *)((&boot_cpu_data)->x86_capability))) : variable_test_bit(((0*32+13)), ((unsigned long *)((&boot_cpu_data)->x86_capability))))))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/tlbflush.h", .line = 51, }; ______r = !!(((__builtin_constant_p((0*32+13)) && ( ((((0*32+13))>>5)==0 && (1UL<<(((0*32+13))&31) & ((1<<((0*32+ 0) & 31))|0|0|(1<<((0*32+ 6) & 31))| (1<<((0*32+ 8) & 31))|0|0|(1<<((0*32+15) & 31))| 0|0))) || ((((0*32+13))>>5)==1 && (1UL<<(((0*32+13))&31) & (0|0))) || ((((0*32+13))>>5)==2 && (1UL<<(((0*32+13))&31) & 0)) || ((((0*32+13))>>5)==3 && (1UL<<(((0*32+13))&31) & (0))) || ((((0*32+13))>>5)==4 && (1UL<<(((0*32+13))&31) & 0)) || ((((0*32+13))>>5)==5 && (1UL<<(((0*32+13))&31) & 0)) || ((((0*32+13))>>5)==6 && (1UL<<(((0*32+13))&31) & 0)) || ((((0*32+13))>>5)==7 && (1UL<<(((0*32+13))&31) & 0)) || ((((0*32+13))>>5)==8 && (1UL<<(((0*32+13))&31) & 0)) || ((((0*32+13))>>5)==9 && (1UL<<(((0*32+13))&31) & 0)) ) ? 1 : (__builtin_constant_p(((0*32+13))) ? constant_test_bit(((0*32+13)), ((unsigned long *)((&boot_cpu_data)->x86_capability))) : variable_test_bit(((0*32+13)), ((unsigned long *)((&boot_cpu_data)->x86_capability))))))); ______f.miss_hit[______r]++; ______r; })) __flush_tlb_global(); else __flush_tlb(); } static inline __attribute__((always_inline)) void __flush_tlb_one(unsigned long addr) { if (__builtin_constant_p(((1))) ? !!((1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/tlbflush.h", .line = 59, }; ______r = !!((1)); ______f.miss_hit[______r]++; ______r; })) __flush_tlb_single(addr); else __flush_tlb(); } extern void flush_tlb_all(void); extern void flush_tlb_current_task(void); extern void flush_tlb_mm(struct mm_struct *); extern void flush_tlb_page(struct vm_area_struct *, unsigned long); static inline __attribute__((always_inline)) void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, unsigned long end) { flush_tlb_mm(vma->vm_mm); } void native_flush_tlb_others(const struct cpumask *cpumask, struct mm_struct *mm, unsigned long va); struct tlb_state { struct mm_struct *active_mm; int state; }; extern __attribute__((section(".data..percpu" ""))) __typeof__(struct tlb_state) cpu_tlbstate __attribute__((__aligned__((1 << (6))))); static inline __attribute__((always_inline)) void reset_lazy_tlbstate(void) { do { typedef typeof(cpu_tlbstate.state) pto_T__; if (__builtin_constant_p(((0))) ? !!((0)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/tlbflush.h", .line = 159, }; ______r = !!((0)); ______f.miss_hit[______r]++; ______r; })) { pto_T__ pto_tmp__; pto_tmp__ = (0); (void)pto_tmp__; } switch (sizeof(cpu_tlbstate.state)) { case 1: asm("mov" "b %1,""%%""fs"":" "%P" "0" : "+m" (cpu_tlbstate.state) : "qi" ((pto_T__)(0))); break; case 2: asm("mov" "w %1,""%%""fs"":" "%P" "0" : "+m" (cpu_tlbstate.state) : "ri" ((pto_T__)(0))); break; case 4: asm("mov" "l %1,""%%""fs"":" "%P" "0" : "+m" (cpu_tlbstate.state) : "ri" ((pto_T__)(0))); break; case 8: asm("mov" "q %1,""%%""fs"":" "%P" "0" : "+m" (cpu_tlbstate.state) : "re" ((pto_T__)(0))); break; default: __bad_percpu_size(); } } while (0); do { typedef typeof(cpu_tlbstate.active_mm) pto_T__; if (__builtin_constant_p(((0))) ? !!((0)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "/data/exp/linux-3.0.4/arch/x86/include/asm/tlbflush.h", .line = 160, }; ______r = !!((0)); ______f.miss_hit[______r]++; ______r; })) { pto_T__ pto_tmp__; pto_tmp__ = (&init_mm); (void)pto_tmp__; } switch (sizeof(cpu_tlbstate.active_mm)) { case 1: asm("mov" "b %1,""%%""fs"":" "%P" "0" : "+m" (cpu_tlbstate.active_mm) : "qi" ((pto_T__)(&init_mm))); break; case 2: asm("mov" "w %1,""%%""fs"":" "%P" "0" : "+m" (cpu_tlbstate.active_mm) : "ri" ((pto_T__)(&init_mm))); break; case 4: asm("mov" "l %1,""%%""fs"":" "%P" "0" : "+m" (cpu_tlbstate.active_mm) : "ri" ((pto_T__)(&init_mm))); break; case 8: asm("mov" "q %1,""%%""fs"":" "%P" "0" : "+m" (cpu_tlbstate.active_mm) : "re" ((pto_T__)(&init_mm))); break; default: __bad_percpu_size(); } } while (0); } static inline __attribute__((always_inline)) void flush_tlb_kernel_range(unsigned long start, unsigned long end) { flush_tlb_all(); } extern unsigned long highstart_pfn, highend_pfn; extern void *kmap_high(struct page *page); extern void kunmap_high(struct page *page); void *kmap(struct page *page); void kunmap(struct page *page); void *kmap_atomic_prot(struct page *page, pgprot_t prot); void *__kmap_atomic(struct page *page); void __kunmap_atomic(void *kvaddr); void *kmap_atomic_pfn(unsigned long pfn); void *kmap_atomic_prot_pfn(unsigned long pfn, pgprot_t prot); struct page *kmap_atomic_to_page(void *ptr); extern void add_highpages_with_active_regions(int nid, unsigned long start_pfn, unsigned long end_pfn); unsigned int nr_free_highpages(void); extern unsigned long totalhigh_pages; void kmap_flush_unused(void); extern __attribute__((section(".data..percpu" ""))) __typeof__(int) __kmap_atomic_idx; static inline __attribute__((always_inline)) int kmap_atomic_idx_push(void) { int idx = ({ typeof(__kmap_atomic_idx) pscr2_ret__; do { const void *__vpp_verify = (typeof(&(__kmap_atomic_idx)))((void *)0); (void)__vpp_verify; } while (0); switch(sizeof(__kmap_atomic_idx)) { case 1: pscr2_ret__ = ({ typeof(__kmap_atomic_idx) paro_ret__ = 1; switch (sizeof(__kmap_atomic_idx)) { case 1: asm("xaddb %0, ""%%""fs"":" "%P" "1" : "+q" (paro_ret__), "+m" (__kmap_atomic_idx) : : "memory"); break; case 2: asm("xaddw %0, ""%%""fs"":" "%P" "1" : "+r" (paro_ret__), "+m" (__kmap_atomic_idx) : : "memory"); break; case 4: asm("xaddl %0, ""%%""fs"":" "%P" "1" : "+r" (paro_ret__), "+m" (__kmap_atomic_idx) : : "memory"); break; case 8: asm("xaddq %0, ""%%""fs"":" "%P" "1" : "+re" (paro_ret__), "+m" (__kmap_atomic_idx) : : "memory"); break; default: __bad_percpu_size(); } paro_ret__ += 1; paro_ret__; }); break; case 2: pscr2_ret__ = ({ typeof(__kmap_atomic_idx) paro_ret__ = 1; switch (sizeof(__kmap_atomic_idx)) { case 1: asm("xaddb %0, ""%%""fs"":" "%P" "1" : "+q" (paro_ret__), "+m" (__kmap_atomic_idx) : : "memory"); break; case 2: asm("xaddw %0, ""%%""fs"":" "%P" "1" : "+r" (paro_ret__), "+m" (__kmap_atomic_idx) : : "memory"); break; case 4: asm("xaddl %0, ""%%""fs"":" "%P" "1" : "+r" (paro_ret__), "+m" (__kmap_atomic_idx) : : "memory"); break; case 8: asm("xaddq %0, ""%%""fs"":" "%P" "1" : "+re" (paro_ret__), "+m" (__kmap_atomic_idx) : : "memory"); break; default: __bad_percpu_size(); } paro_ret__ += 1; paro_ret__; }); break; case 4: pscr2_ret__ = ({ typeof(__kmap_atomic_idx) paro_ret__ = 1; switch (sizeof(__kmap_atomic_idx)) { case 1: asm("xaddb %0, ""%%""fs"":" "%P" "1" : "+q" (paro_ret__), "+m" (__kmap_atomic_idx) : : "memory"); break; case 2: asm("xaddw %0, ""%%""fs"":" "%P" "1" : "+r" (paro_ret__), "+m" (__kmap_atomic_idx) : : "memory"); break; case 4: asm("xaddl %0, ""%%""fs"":" "%P" "1" : "+r" (paro_ret__), "+m" (__kmap_atomic_idx) : : "memory"); break; case 8: asm("xaddq %0, ""%%""fs"":" "%P" "1" : "+re" (paro_ret__), "+m" (__kmap_atomic_idx) : : "memory"); break; default: __bad_percpu_size(); } paro_ret__ += 1; paro_ret__; }); break; case 8: pscr2_ret__ = ({ typeof(__kmap_atomic_idx) ret__; do { add_preempt_count(1); __asm__ __volatile__("": : :"memory"); } while (0); do { do { const void *__vpp_verify = (typeof(&((__kmap_atomic_idx))))((void *)0); (void)__vpp_verify; } while (0); switch(sizeof((__kmap_atomic_idx))) { case 1: do { typedef typeof(((__kmap_atomic_idx))) pao_T__; const int pao_ID__ = (__builtin_constant_p((1)) && (((1)) == 1 || ((1)) == -1)) ? ((1)) : 0; if (__builtin_constant_p(((0))) ? !!((0)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 84, }; ______r = !!((0)); ______f.miss_hit[______r]++; ______r; })) { pao_T__ pao_tmp__; pao_tmp__ = ((1)); (void)pao_tmp__; } switch (sizeof(((__kmap_atomic_idx)))) { case 1: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 84, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incb ""%%""fs"":" "%P" "0" : "+m" (((__kmap_atomic_idx)))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 84, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decb ""%%""fs"":" "%P" "0" : "+m" (((__kmap_atomic_idx)))); else asm("addb %1, ""%%""fs"":" "%P" "0" : "+m" (((__kmap_atomic_idx))) : "qi" ((pao_T__)((1)))); break; case 2: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 84, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incw ""%%""fs"":" "%P" "0" : "+m" (((__kmap_atomic_idx)))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 84, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decw ""%%""fs"":" "%P" "0" : "+m" (((__kmap_atomic_idx)))); else asm("addw %1, ""%%""fs"":" "%P" "0" : "+m" (((__kmap_atomic_idx))) : "ri" ((pao_T__)((1)))); break; case 4: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 84, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incl ""%%""fs"":" "%P" "0" : "+m" (((__kmap_atomic_idx)))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 84, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decl ""%%""fs"":" "%P" "0" : "+m" (((__kmap_atomic_idx)))); else asm("addl %1, ""%%""fs"":" "%P" "0" : "+m" (((__kmap_atomic_idx))) : "ri" ((pao_T__)((1)))); break; case 8: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 84, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incq ""%%""fs"":" "%P" "0" : "+m" (((__kmap_atomic_idx)))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 84, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decq ""%%""fs"":" "%P" "0" : "+m" (((__kmap_atomic_idx)))); else asm("addq %1, ""%%""fs"":" "%P" "0" : "+m" (((__kmap_atomic_idx))) : "re" ((pao_T__)((1)))); break; default: __bad_percpu_size(); } } while (0);break; case 2: do { typedef typeof(((__kmap_atomic_idx))) pao_T__; const int pao_ID__ = (__builtin_constant_p((1)) && (((1)) == 1 || ((1)) == -1)) ? ((1)) : 0; if (__builtin_constant_p(((0))) ? !!((0)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 84, }; ______r = !!((0)); ______f.miss_hit[______r]++; ______r; })) { pao_T__ pao_tmp__; pao_tmp__ = ((1)); (void)pao_tmp__; } switch (sizeof(((__kmap_atomic_idx)))) { case 1: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 84, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incb ""%%""fs"":" "%P" "0" : "+m" (((__kmap_atomic_idx)))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 84, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decb ""%%""fs"":" "%P" "0" : "+m" (((__kmap_atomic_idx)))); else asm("addb %1, ""%%""fs"":" "%P" "0" : "+m" (((__kmap_atomic_idx))) : "qi" ((pao_T__)((1)))); break; case 2: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 84, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incw ""%%""fs"":" "%P" "0" : "+m" (((__kmap_atomic_idx)))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 84, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decw ""%%""fs"":" "%P" "0" : "+m" (((__kmap_atomic_idx)))); else asm("addw %1, ""%%""fs"":" "%P" "0" : "+m" (((__kmap_atomic_idx))) : "ri" ((pao_T__)((1)))); break; case 4: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 84, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incl ""%%""fs"":" "%P" "0" : "+m" (((__kmap_atomic_idx)))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 84, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decl ""%%""fs"":" "%P" "0" : "+m" (((__kmap_atomic_idx)))); else asm("addl %1, ""%%""fs"":" "%P" "0" : "+m" (((__kmap_atomic_idx))) : "ri" ((pao_T__)((1)))); break; case 8: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 84, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incq ""%%""fs"":" "%P" "0" : "+m" (((__kmap_atomic_idx)))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 84, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decq ""%%""fs"":" "%P" "0" : "+m" (((__kmap_atomic_idx)))); else asm("addq %1, ""%%""fs"":" "%P" "0" : "+m" (((__kmap_atomic_idx))) : "re" ((pao_T__)((1)))); break; default: __bad_percpu_size(); } } while (0);break; case 4: do { typedef typeof(((__kmap_atomic_idx))) pao_T__; const int pao_ID__ = (__builtin_constant_p((1)) && (((1)) == 1 || ((1)) == -1)) ? ((1)) : 0; if (__builtin_constant_p(((0))) ? !!((0)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 84, }; ______r = !!((0)); ______f.miss_hit[______r]++; ______r; })) { pao_T__ pao_tmp__; pao_tmp__ = ((1)); (void)pao_tmp__; } switch (sizeof(((__kmap_atomic_idx)))) { case 1: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 84, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incb ""%%""fs"":" "%P" "0" : "+m" (((__kmap_atomic_idx)))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 84, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decb ""%%""fs"":" "%P" "0" : "+m" (((__kmap_atomic_idx)))); else asm("addb %1, ""%%""fs"":" "%P" "0" : "+m" (((__kmap_atomic_idx))) : "qi" ((pao_T__)((1)))); break; case 2: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 84, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incw ""%%""fs"":" "%P" "0" : "+m" (((__kmap_atomic_idx)))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 84, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decw ""%%""fs"":" "%P" "0" : "+m" (((__kmap_atomic_idx)))); else asm("addw %1, ""%%""fs"":" "%P" "0" : "+m" (((__kmap_atomic_idx))) : "ri" ((pao_T__)((1)))); break; case 4: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 84, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incl ""%%""fs"":" "%P" "0" : "+m" (((__kmap_atomic_idx)))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 84, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decl ""%%""fs"":" "%P" "0" : "+m" (((__kmap_atomic_idx)))); else asm("addl %1, ""%%""fs"":" "%P" "0" : "+m" (((__kmap_atomic_idx))) : "ri" ((pao_T__)((1)))); break; case 8: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 84, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incq ""%%""fs"":" "%P" "0" : "+m" (((__kmap_atomic_idx)))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 84, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decq ""%%""fs"":" "%P" "0" : "+m" (((__kmap_atomic_idx)))); else asm("addq %1, ""%%""fs"":" "%P" "0" : "+m" (((__kmap_atomic_idx))) : "re" ((pao_T__)((1)))); break; default: __bad_percpu_size(); } } while (0);break; case 8: do { *({ unsigned long tcp_ptr__; do { const void *__vpp_verify = (typeof(&(((__kmap_atomic_idx)))))((void *)0); (void)__vpp_verify; } while (0); asm volatile("add " "%%""fs"":" "%P" "1" ", %0" : "=r" (tcp_ptr__) : "m" (this_cpu_off), "0" (&(((__kmap_atomic_idx))))); (typeof(*(&(((__kmap_atomic_idx))))) *)tcp_ptr__; }) += ((1)); } while (0);break; default: __bad_size_call_parameter();break; } } while (0); ret__ = ({ typeof((__kmap_atomic_idx)) pscr_ret__; do { const void *__vpp_verify = (typeof(&((__kmap_atomic_idx))))((void *)0); (void)__vpp_verify; } while (0); switch(sizeof((__kmap_atomic_idx))) { case 1: pscr_ret__ = ({ typeof(((__kmap_atomic_idx))) pfo_ret__; switch (sizeof(((__kmap_atomic_idx)))) { case 1: asm("mov" "b ""%%""fs"":" "%P" "1"",%0" : "=q" (pfo_ret__) : "m"((__kmap_atomic_idx))); break; case 2: asm("mov" "w ""%%""fs"":" "%P" "1"",%0" : "=r" (pfo_ret__) : "m"((__kmap_atomic_idx))); break; case 4: asm("mov" "l ""%%""fs"":" "%P" "1"",%0" : "=r" (pfo_ret__) : "m"((__kmap_atomic_idx))); break; case 8: asm("mov" "q ""%%""fs"":" "%P" "1"",%0" : "=r" (pfo_ret__) : "m"((__kmap_atomic_idx))); break; default: __bad_percpu_size(); } pfo_ret__; });break; case 2: pscr_ret__ = ({ typeof(((__kmap_atomic_idx))) pfo_ret__; switch (sizeof(((__kmap_atomic_idx)))) { case 1: asm("mov" "b ""%%""fs"":" "%P" "1"",%0" : "=q" (pfo_ret__) : "m"((__kmap_atomic_idx))); break; case 2: asm("mov" "w ""%%""fs"":" "%P" "1"",%0" : "=r" (pfo_ret__) : "m"((__kmap_atomic_idx))); break; case 4: asm("mov" "l ""%%""fs"":" "%P" "1"",%0" : "=r" (pfo_ret__) : "m"((__kmap_atomic_idx))); break; case 8: asm("mov" "q ""%%""fs"":" "%P" "1"",%0" : "=r" (pfo_ret__) : "m"((__kmap_atomic_idx))); break; default: __bad_percpu_size(); } pfo_ret__; });break; case 4: pscr_ret__ = ({ typeof(((__kmap_atomic_idx))) pfo_ret__; switch (sizeof(((__kmap_atomic_idx)))) { case 1: asm("mov" "b ""%%""fs"":" "%P" "1"",%0" : "=q" (pfo_ret__) : "m"((__kmap_atomic_idx))); break; case 2: asm("mov" "w ""%%""fs"":" "%P" "1"",%0" : "=r" (pfo_ret__) : "m"((__kmap_atomic_idx))); break; case 4: asm("mov" "l ""%%""fs"":" "%P" "1"",%0" : "=r" (pfo_ret__) : "m"((__kmap_atomic_idx))); break; case 8: asm("mov" "q ""%%""fs"":" "%P" "1"",%0" : "=r" (pfo_ret__) : "m"((__kmap_atomic_idx))); break; default: __bad_percpu_size(); } pfo_ret__; });break; case 8: pscr_ret__ = (*({ unsigned long tcp_ptr__; do { const void *__vpp_verify = (typeof(&((__kmap_atomic_idx))))((void *)0); (void)__vpp_verify; } while (0); asm volatile("add " "%%""fs"":" "%P" "1" ", %0" : "=r" (tcp_ptr__) : "m" (this_cpu_off), "0" (&((__kmap_atomic_idx)))); (typeof(*(&((__kmap_atomic_idx)))) *)tcp_ptr__; }));break; default: __bad_size_call_parameter();break; } pscr_ret__; }); do { do { __asm__ __volatile__("": : :"memory"); sub_preempt_count(1); } while (0); __asm__ __volatile__("": : :"memory"); do { if (__builtin_constant_p((((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 84, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 84, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 84, }; ______r = !!(((__builtin_constant_p(test_ti_thread_flag(current_thread_info(), 3)) ? !!(test_ti_thread_flag(current_thread_info(), 3)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 84, }; ______r = __builtin_expect(!!(test_ti_thread_flag(current_thread_info(), 3)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) preempt_schedule(); } while (0); } while (0); ret__; }); break; default: __bad_size_call_parameter(); break; } pscr2_ret__; }) - 1; return idx; } static inline __attribute__((always_inline)) int kmap_atomic_idx(void) { return ({ typeof((__kmap_atomic_idx)) pscr_ret__; do { const void *__vpp_verify = (typeof(&((__kmap_atomic_idx))))((void *)0); (void)__vpp_verify; } while (0); switch(sizeof((__kmap_atomic_idx))) { case 1: pscr_ret__ = ({ typeof(((__kmap_atomic_idx))) pfo_ret__; switch (sizeof(((__kmap_atomic_idx)))) { case 1: asm("mov" "b ""%%""fs"":" "%P" "1"",%0" : "=q" (pfo_ret__) : "m"((__kmap_atomic_idx))); break; case 2: asm("mov" "w ""%%""fs"":" "%P" "1"",%0" : "=r" (pfo_ret__) : "m"((__kmap_atomic_idx))); break; case 4: asm("mov" "l ""%%""fs"":" "%P" "1"",%0" : "=r" (pfo_ret__) : "m"((__kmap_atomic_idx))); break; case 8: asm("mov" "q ""%%""fs"":" "%P" "1"",%0" : "=r" (pfo_ret__) : "m"((__kmap_atomic_idx))); break; default: __bad_percpu_size(); } pfo_ret__; });break; case 2: pscr_ret__ = ({ typeof(((__kmap_atomic_idx))) pfo_ret__; switch (sizeof(((__kmap_atomic_idx)))) { case 1: asm("mov" "b ""%%""fs"":" "%P" "1"",%0" : "=q" (pfo_ret__) : "m"((__kmap_atomic_idx))); break; case 2: asm("mov" "w ""%%""fs"":" "%P" "1"",%0" : "=r" (pfo_ret__) : "m"((__kmap_atomic_idx))); break; case 4: asm("mov" "l ""%%""fs"":" "%P" "1"",%0" : "=r" (pfo_ret__) : "m"((__kmap_atomic_idx))); break; case 8: asm("mov" "q ""%%""fs"":" "%P" "1"",%0" : "=r" (pfo_ret__) : "m"((__kmap_atomic_idx))); break; default: __bad_percpu_size(); } pfo_ret__; });break; case 4: pscr_ret__ = ({ typeof(((__kmap_atomic_idx))) pfo_ret__; switch (sizeof(((__kmap_atomic_idx)))) { case 1: asm("mov" "b ""%%""fs"":" "%P" "1"",%0" : "=q" (pfo_ret__) : "m"((__kmap_atomic_idx))); break; case 2: asm("mov" "w ""%%""fs"":" "%P" "1"",%0" : "=r" (pfo_ret__) : "m"((__kmap_atomic_idx))); break; case 4: asm("mov" "l ""%%""fs"":" "%P" "1"",%0" : "=r" (pfo_ret__) : "m"((__kmap_atomic_idx))); break; case 8: asm("mov" "q ""%%""fs"":" "%P" "1"",%0" : "=r" (pfo_ret__) : "m"((__kmap_atomic_idx))); break; default: __bad_percpu_size(); } pfo_ret__; });break; case 8: pscr_ret__ = (*({ unsigned long tcp_ptr__; do { const void *__vpp_verify = (typeof(&((__kmap_atomic_idx))))((void *)0); (void)__vpp_verify; } while (0); asm volatile("add " "%%""fs"":" "%P" "1" ", %0" : "=r" (tcp_ptr__) : "m" (this_cpu_off), "0" (&((__kmap_atomic_idx)))); (typeof(*(&((__kmap_atomic_idx)))) *)tcp_ptr__; }));break; default: __bad_size_call_parameter();break; } pscr_ret__; }) - 1; } static inline __attribute__((always_inline)) void kmap_atomic_idx_pop(void) { do { do { const void *__vpp_verify = (typeof(&((((__kmap_atomic_idx))))))((void *)0); (void)__vpp_verify; } while (0); switch(sizeof((((__kmap_atomic_idx))))) { case 1: do { typedef typeof(((((__kmap_atomic_idx))))) pao_T__; const int pao_ID__ = (__builtin_constant_p((-(1))) && (((-(1))) == 1 || ((-(1))) == -1)) ? ((-(1))) : 0; if (__builtin_constant_p(((0))) ? !!((0)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 105, }; ______r = !!((0)); ______f.miss_hit[______r]++; ______r; })) { pao_T__ pao_tmp__; pao_tmp__ = ((-(1))); (void)pao_tmp__; } switch (sizeof(((((__kmap_atomic_idx)))))) { case 1: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 105, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incb ""%%""fs"":" "%P" "0" : "+m" (((((__kmap_atomic_idx)))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 105, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decb ""%%""fs"":" "%P" "0" : "+m" (((((__kmap_atomic_idx)))))); else asm("addb %1, ""%%""fs"":" "%P" "0" : "+m" (((((__kmap_atomic_idx))))) : "qi" ((pao_T__)((-(1))))); break; case 2: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 105, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incw ""%%""fs"":" "%P" "0" : "+m" (((((__kmap_atomic_idx)))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 105, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decw ""%%""fs"":" "%P" "0" : "+m" (((((__kmap_atomic_idx)))))); else asm("addw %1, ""%%""fs"":" "%P" "0" : "+m" (((((__kmap_atomic_idx))))) : "ri" ((pao_T__)((-(1))))); break; case 4: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 105, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incl ""%%""fs"":" "%P" "0" : "+m" (((((__kmap_atomic_idx)))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 105, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decl ""%%""fs"":" "%P" "0" : "+m" (((((__kmap_atomic_idx)))))); else asm("addl %1, ""%%""fs"":" "%P" "0" : "+m" (((((__kmap_atomic_idx))))) : "ri" ((pao_T__)((-(1))))); break; case 8: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 105, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incq ""%%""fs"":" "%P" "0" : "+m" (((((__kmap_atomic_idx)))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 105, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decq ""%%""fs"":" "%P" "0" : "+m" (((((__kmap_atomic_idx)))))); else asm("addq %1, ""%%""fs"":" "%P" "0" : "+m" (((((__kmap_atomic_idx))))) : "re" ((pao_T__)((-(1))))); break; default: __bad_percpu_size(); } } while (0);break; case 2: do { typedef typeof(((((__kmap_atomic_idx))))) pao_T__; const int pao_ID__ = (__builtin_constant_p((-(1))) && (((-(1))) == 1 || ((-(1))) == -1)) ? ((-(1))) : 0; if (__builtin_constant_p(((0))) ? !!((0)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 105, }; ______r = !!((0)); ______f.miss_hit[______r]++; ______r; })) { pao_T__ pao_tmp__; pao_tmp__ = ((-(1))); (void)pao_tmp__; } switch (sizeof(((((__kmap_atomic_idx)))))) { case 1: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 105, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incb ""%%""fs"":" "%P" "0" : "+m" (((((__kmap_atomic_idx)))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 105, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decb ""%%""fs"":" "%P" "0" : "+m" (((((__kmap_atomic_idx)))))); else asm("addb %1, ""%%""fs"":" "%P" "0" : "+m" (((((__kmap_atomic_idx))))) : "qi" ((pao_T__)((-(1))))); break; case 2: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 105, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incw ""%%""fs"":" "%P" "0" : "+m" (((((__kmap_atomic_idx)))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 105, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decw ""%%""fs"":" "%P" "0" : "+m" (((((__kmap_atomic_idx)))))); else asm("addw %1, ""%%""fs"":" "%P" "0" : "+m" (((((__kmap_atomic_idx))))) : "ri" ((pao_T__)((-(1))))); break; case 4: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 105, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incl ""%%""fs"":" "%P" "0" : "+m" (((((__kmap_atomic_idx)))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 105, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decl ""%%""fs"":" "%P" "0" : "+m" (((((__kmap_atomic_idx)))))); else asm("addl %1, ""%%""fs"":" "%P" "0" : "+m" (((((__kmap_atomic_idx))))) : "ri" ((pao_T__)((-(1))))); break; case 8: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 105, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incq ""%%""fs"":" "%P" "0" : "+m" (((((__kmap_atomic_idx)))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 105, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decq ""%%""fs"":" "%P" "0" : "+m" (((((__kmap_atomic_idx)))))); else asm("addq %1, ""%%""fs"":" "%P" "0" : "+m" (((((__kmap_atomic_idx))))) : "re" ((pao_T__)((-(1))))); break; default: __bad_percpu_size(); } } while (0);break; case 4: do { typedef typeof(((((__kmap_atomic_idx))))) pao_T__; const int pao_ID__ = (__builtin_constant_p((-(1))) && (((-(1))) == 1 || ((-(1))) == -1)) ? ((-(1))) : 0; if (__builtin_constant_p(((0))) ? !!((0)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 105, }; ______r = !!((0)); ______f.miss_hit[______r]++; ______r; })) { pao_T__ pao_tmp__; pao_tmp__ = ((-(1))); (void)pao_tmp__; } switch (sizeof(((((__kmap_atomic_idx)))))) { case 1: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 105, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incb ""%%""fs"":" "%P" "0" : "+m" (((((__kmap_atomic_idx)))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 105, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decb ""%%""fs"":" "%P" "0" : "+m" (((((__kmap_atomic_idx)))))); else asm("addb %1, ""%%""fs"":" "%P" "0" : "+m" (((((__kmap_atomic_idx))))) : "qi" ((pao_T__)((-(1))))); break; case 2: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 105, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incw ""%%""fs"":" "%P" "0" : "+m" (((((__kmap_atomic_idx)))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 105, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decw ""%%""fs"":" "%P" "0" : "+m" (((((__kmap_atomic_idx)))))); else asm("addw %1, ""%%""fs"":" "%P" "0" : "+m" (((((__kmap_atomic_idx))))) : "ri" ((pao_T__)((-(1))))); break; case 4: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 105, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incl ""%%""fs"":" "%P" "0" : "+m" (((((__kmap_atomic_idx)))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 105, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decl ""%%""fs"":" "%P" "0" : "+m" (((((__kmap_atomic_idx)))))); else asm("addl %1, ""%%""fs"":" "%P" "0" : "+m" (((((__kmap_atomic_idx))))) : "ri" ((pao_T__)((-(1))))); break; case 8: if (__builtin_constant_p(((pao_ID__ == 1))) ? !!((pao_ID__ == 1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 105, }; ______r = !!((pao_ID__ == 1)); ______f.miss_hit[______r]++; ______r; })) asm("incq ""%%""fs"":" "%P" "0" : "+m" (((((__kmap_atomic_idx)))))); else if (__builtin_constant_p(((pao_ID__ == -1))) ? !!((pao_ID__ == -1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 105, }; ______r = !!((pao_ID__ == -1)); ______f.miss_hit[______r]++; ______r; })) asm("decq ""%%""fs"":" "%P" "0" : "+m" (((((__kmap_atomic_idx)))))); else asm("addq %1, ""%%""fs"":" "%P" "0" : "+m" (((((__kmap_atomic_idx))))) : "re" ((pao_T__)((-(1))))); break; default: __bad_percpu_size(); } } while (0);break; case 8: do { *({ unsigned long tcp_ptr__; do { const void *__vpp_verify = (typeof(&(((((__kmap_atomic_idx)))))))((void *)0); (void)__vpp_verify; } while (0); asm volatile("add " "%%""fs"":" "%P" "1" ", %0" : "=r" (tcp_ptr__) : "m" (this_cpu_off), "0" (&(((((__kmap_atomic_idx))))))); (typeof(*(&(((((__kmap_atomic_idx))))))) *)tcp_ptr__; }) += ((-(1))); } while (0);break; default: __bad_size_call_parameter();break; } } while (0); } static inline __attribute__((always_inline)) void clear_user_highpage(struct page *page, unsigned long vaddr) { void *addr = __kmap_atomic(page); clear_user_page(addr, vaddr, page); do { do { ((void)sizeof(char[1 - 2*!!(__builtin_types_compatible_p(typeof((addr)), typeof(struct page *)))])); if (__builtin_constant_p(((__builtin_types_compatible_p(typeof((addr)), typeof(struct page *))))) ? !!((__builtin_types_compatible_p(typeof((addr)), typeof(struct page *)))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 132, }; ______r = !!((__builtin_types_compatible_p(typeof((addr)), typeof(struct page *)))); ______f.miss_hit[______r]++; ______r; })) __build_bug_on_failed = 1; } while(0); __kunmap_atomic(addr); } while (0); } static inline __attribute__((always_inline)) struct page * alloc_zeroed_user_highpage_movable(struct vm_area_struct *vma, unsigned long vaddr) { return alloc_pages_node(numa_node_id(), ((( gfp_t)0x10u) | (( gfp_t)0x40u) | (( gfp_t)0x80u) | (( gfp_t)0x20000u) | (( gfp_t)0x02u)) | (( gfp_t)0x8000u) | (( gfp_t)0x08u), 0); } static inline __attribute__((always_inline)) void clear_highpage(struct page *page) { void *kaddr = __kmap_atomic(page); clear_page(kaddr); do { do { ((void)sizeof(char[1 - 2*!!(__builtin_types_compatible_p(typeof((kaddr)), typeof(struct page *)))])); if (__builtin_constant_p(((__builtin_types_compatible_p(typeof((kaddr)), typeof(struct page *))))) ? !!((__builtin_types_compatible_p(typeof((kaddr)), typeof(struct page *)))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 185, }; ______r = !!((__builtin_types_compatible_p(typeof((kaddr)), typeof(struct page *)))); ______f.miss_hit[______r]++; ______r; })) __build_bug_on_failed = 1; } while(0); __kunmap_atomic(kaddr); } while (0); } static inline __attribute__((always_inline)) void zero_user_segments(struct page *page, unsigned start1, unsigned end1, unsigned start2, unsigned end2) { void *kaddr = __kmap_atomic(page); do { if (__builtin_constant_p((((__builtin_constant_p(end1 > ((1UL) << 12) || end2 > ((1UL) << 12)) ? !!(end1 > ((1UL) << 12) || end2 > ((1UL) << 12)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 194, }; ______r = __builtin_expect(!!(end1 > ((1UL) << 12) || end2 > ((1UL) << 12)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p(end1 > ((1UL) << 12) || end2 > ((1UL) << 12)) ? !!(end1 > ((1UL) << 12) || end2 > ((1UL) << 12)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 194, }; ______r = __builtin_expect(!!(end1 > ((1UL) << 12) || end2 > ((1UL) << 12)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 194, }; ______r = !!(((__builtin_constant_p(end1 > ((1UL) << 12) || end2 > ((1UL) << 12)) ? !!(end1 > ((1UL) << 12) || end2 > ((1UL) << 12)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 194, }; ______r = __builtin_expect(!!(end1 > ((1UL) << 12) || end2 > ((1UL) << 12)), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) do { asm volatile("1:\tud2\n" ".pushsection __bug_table,\"a\"\n" "2:\t.long 1b, %c0\n" "\t.word %c1, 0\n" "\t.org 2b+%c2\n" ".popsection" : : "i" ("include/linux/highmem.h"), "i" (194), "i" (sizeof(struct bug_entry))); __builtin_unreachable(); } while (0); } while(0); if (__builtin_constant_p(((end1 > start1))) ? !!((end1 > start1)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 196, }; ______r = !!((end1 > start1)); ______f.miss_hit[______r]++; ______r; })) __builtin_memset(kaddr + start1, 0, end1 - start1); if (__builtin_constant_p(((end2 > start2))) ? !!((end2 > start2)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 199, }; ______r = !!((end2 > start2)); ______f.miss_hit[______r]++; ______r; })) __builtin_memset(kaddr + start2, 0, end2 - start2); do { do { ((void)sizeof(char[1 - 2*!!(__builtin_types_compatible_p(typeof((kaddr)), typeof(struct page *)))])); if (__builtin_constant_p(((__builtin_types_compatible_p(typeof((kaddr)), typeof(struct page *))))) ? !!((__builtin_types_compatible_p(typeof((kaddr)), typeof(struct page *)))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 202, }; ______r = !!((__builtin_types_compatible_p(typeof((kaddr)), typeof(struct page *)))); ______f.miss_hit[______r]++; ______r; })) __build_bug_on_failed = 1; } while(0); __kunmap_atomic(kaddr); } while (0); do { } while (0); } static inline __attribute__((always_inline)) void zero_user_segment(struct page *page, unsigned start, unsigned end) { zero_user_segments(page, start, end, 0, 0); } static inline __attribute__((always_inline)) void zero_user(struct page *page, unsigned start, unsigned size) { zero_user_segments(page, start, start + size, 0, 0); } static inline __attribute__((always_inline)) void __attribute__((deprecated)) memclear_highpage_flush(struct page *page, unsigned int offset, unsigned int size) { zero_user(page, offset, size); } static inline __attribute__((always_inline)) void copy_user_highpage(struct page *to, struct page *from, unsigned long vaddr, struct vm_area_struct *vma) { char *vfrom, *vto; vfrom = __kmap_atomic(from); vto = __kmap_atomic(to); copy_user_page(vto, vfrom, vaddr, to); do { do { ((void)sizeof(char[1 - 2*!!(__builtin_types_compatible_p(typeof((vto)), typeof(struct page *)))])); if (__builtin_constant_p(((__builtin_types_compatible_p(typeof((vto)), typeof(struct page *))))) ? !!((__builtin_types_compatible_p(typeof((vto)), typeof(struct page *)))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 234, }; ______r = !!((__builtin_types_compatible_p(typeof((vto)), typeof(struct page *)))); ______f.miss_hit[______r]++; ______r; })) __build_bug_on_failed = 1; } while(0); __kunmap_atomic(vto); } while (0); do { do { ((void)sizeof(char[1 - 2*!!(__builtin_types_compatible_p(typeof((vfrom)), typeof(struct page *)))])); if (__builtin_constant_p(((__builtin_types_compatible_p(typeof((vfrom)), typeof(struct page *))))) ? !!((__builtin_types_compatible_p(typeof((vfrom)), typeof(struct page *)))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 235, }; ______r = !!((__builtin_types_compatible_p(typeof((vfrom)), typeof(struct page *)))); ______f.miss_hit[______r]++; ______r; })) __build_bug_on_failed = 1; } while(0); __kunmap_atomic(vfrom); } while (0); } static inline __attribute__((always_inline)) void copy_highpage(struct page *to, struct page *from) { char *vfrom, *vto; vfrom = __kmap_atomic(from); vto = __kmap_atomic(to); copy_page(vto, vfrom); do { do { ((void)sizeof(char[1 - 2*!!(__builtin_types_compatible_p(typeof((vto)), typeof(struct page *)))])); if (__builtin_constant_p(((__builtin_types_compatible_p(typeof((vto)), typeof(struct page *))))) ? !!((__builtin_types_compatible_p(typeof((vto)), typeof(struct page *)))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 247, }; ______r = !!((__builtin_types_compatible_p(typeof((vto)), typeof(struct page *)))); ______f.miss_hit[______r]++; ______r; })) __build_bug_on_failed = 1; } while(0); __kunmap_atomic(vto); } while (0); do { do { ((void)sizeof(char[1 - 2*!!(__builtin_types_compatible_p(typeof((vfrom)), typeof(struct page *)))])); if (__builtin_constant_p(((__builtin_types_compatible_p(typeof((vfrom)), typeof(struct page *))))) ? !!((__builtin_types_compatible_p(typeof((vfrom)), typeof(struct page *)))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/highmem.h", .line = 248, }; ______r = !!((__builtin_types_compatible_p(typeof((vfrom)), typeof(struct page *)))); ______f.miss_hit[______r]++; ______r; })) __build_bug_on_failed = 1; } while(0); __kunmap_atomic(vfrom); } while (0); } struct scatterlist { unsigned long page_link; unsigned int offset; unsigned int length; dma_addr_t dma_address; unsigned int dma_length; }; struct sg_table { struct scatterlist *sgl; unsigned int nents; unsigned int orig_nents; }; static inline __attribute__((always_inline)) void sg_assign_page(struct scatterlist *sg, struct page *page) { unsigned long page_link = sg->page_link & 0x3; do { if (__builtin_constant_p((((__builtin_constant_p((unsigned long) page & 0x03) ? !!((unsigned long) page & 0x03) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/scatterlist.h", .line = 63, }; ______r = __builtin_expect(!!((unsigned long) page & 0x03), 1); ftrace_likely_update(&______f, ______r, 0); ______r; }))))) ? !!(((__builtin_constant_p((unsigned long) page & 0x03) ? !!((unsigned long) page & 0x03) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/scatterlist.h", .line = 63, }; ______r = __builtin_expect(!!((unsigned long) page & 0x03), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/linux/scatterlist.h", .line = 63, }; ______r = !!(((__builtin_constant_p((unsigned long) page & 0x03) ? !!((unsigned long) page & 0x03) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/linux/scatterlist.h", .line = 63, }; ______r = __builtin_expect(!!((unsigned long) page & 0x03), 1); ftrace_likely_update(&______f, ______r, 0); ______r; })))); ______f.miss_hit[______r]++; ______r; })) do { asm volatile("1:\tud2\n" ".pushsection __bug_table,\"a\"\n" "2:\t.long 1b, %c0\n" "\t.word %c1, 0\n" "\t.org 2b+%c2\n" ".popsection" : : "i" ("include/linux/scatterlist.h"), "i" (63), "i" (sizeof(struct bug_entry))); __builtin_unreachable(); } while (0); } while(0); sg->page_link = page_link | (unsigned long) page; } static inline __attribute__((always_inline)) void sg_set_page(struct scatterlist *sg, struct page *page, unsigned int len, unsigned int offset) { sg_assign_page(sg, page); sg->offset = offset; sg->length = len; } static inline __attribute__((always_inline)) struct page *sg_page(struct scatterlist *sg) { return (struct page *)((sg)->page_link & ~0x3); } static inline __attribute__((always_inline)) void sg_set_buf(struct scatterlist *sg, const void *buf, unsigned int buflen) { sg_set_page(sg, (mem_map + (((((unsigned long)(buf)) - ((unsigned long)(0xC0000000UL))) >> 12) - (0UL))), buflen, ((unsigned long)(buf) & ~(~(((1UL) << 12)-1)))); } static inline __attribute__((always_inline)) void sg_chain(struct scatterlist *prv, unsigned int prv_nents, struct scatterlist *sgl) { prv[prv_nents - 1].offset = 0; prv[prv_nents - 1].length = 0; prv[prv_nents - 1].page_link = ((unsigned long) sgl | 0x01) & ~0x02; } static inline __attribute__((always_inline)) void sg_mark_end(struct scatterlist *sg) { sg->page_link |= 0x02; sg->page_link &= ~0x01; } static inline __attribute__((always_inline)) dma_addr_t sg_phys(struct scatterlist *sg) { return ((dma_addr_t)((unsigned long)((sg_page(sg)) - mem_map) + (0UL)) << 12) + sg->offset; } static inline __attribute__((always_inline)) void *sg_virt(struct scatterlist *sg) { return page_address(sg_page(sg)) + sg->offset; } struct scatterlist *sg_next(struct scatterlist *); struct scatterlist *sg_last(struct scatterlist *s, unsigned int); void sg_init_table(struct scatterlist *, unsigned int); void sg_init_one(struct scatterlist *, const void *, unsigned int); typedef struct scatterlist *(sg_alloc_fn)(unsigned int, gfp_t); typedef void (sg_free_fn)(struct scatterlist *, unsigned int); void __sg_free_table(struct sg_table *, unsigned int, sg_free_fn *); void sg_free_table(struct sg_table *); int __sg_alloc_table(struct sg_table *, unsigned int, unsigned int, gfp_t, sg_alloc_fn *); int sg_alloc_table(struct sg_table *, unsigned int, gfp_t); size_t sg_copy_from_buffer(struct scatterlist *sgl, unsigned int nents, void *buf, size_t buflen); size_t sg_copy_to_buffer(struct scatterlist *sgl, unsigned int nents, void *buf, size_t buflen); struct sg_mapping_iter { struct page *page; void *addr; size_t length; size_t consumed; struct scatterlist *__sg; unsigned int __nents; unsigned int __offset; unsigned int __flags; }; void sg_miter_start(struct sg_mapping_iter *miter, struct scatterlist *sgl, unsigned int nents, unsigned int flags); bool sg_miter_next(struct sg_mapping_iter *miter); void sg_miter_stop(struct sg_mapping_iter *miter); static inline __attribute__((always_inline)) enum km_type crypto_kmap_type(int out) { enum km_type type; if (__builtin_constant_p((((((current_thread_info()->preempt_count) & (((1UL << (8))-1) << (0 + 8))))))) ? !!(((((current_thread_info()->preempt_count) & (((1UL << (8))-1) << (0 + 8)))))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/crypto/scatterwalk.h", .line = 32, }; ______r = !!(((((current_thread_info()->preempt_count) & (((1UL << (8))-1) << (0 + 8)))))); ______f.miss_hit[______r]++; ______r; })) type = out * (KM_SOFTIRQ1 - KM_SOFTIRQ0) + KM_SOFTIRQ0; else type = out * (KM_USER1 - KM_USER0) + KM_USER0; return type; } static inline __attribute__((always_inline)) void *crypto_kmap(struct page *page, int out) { return __kmap_atomic(page); } static inline __attribute__((always_inline)) void crypto_kunmap(void *vaddr, int out) { do { do { ((void)sizeof(char[1 - 2*!!(__builtin_types_compatible_p(typeof((vaddr)), typeof(struct page *)))])); if (__builtin_constant_p(((__builtin_types_compatible_p(typeof((vaddr)), typeof(struct page *))))) ? !!((__builtin_types_compatible_p(typeof((vaddr)), typeof(struct page *)))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/crypto/scatterwalk.h", .line = 47, }; ______r = !!((__builtin_types_compatible_p(typeof((vaddr)), typeof(struct page *)))); ______f.miss_hit[______r]++; ______r; })) __build_bug_on_failed = 1; } while(0); __kunmap_atomic(vaddr); } while (0); } static inline __attribute__((always_inline)) void crypto_yield(u32 flags) { if (__builtin_constant_p(((flags & 0x00000200))) ? !!((flags & 0x00000200)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/crypto/scatterwalk.h", .line = 52, }; ______r = !!((flags & 0x00000200)); ______f.miss_hit[______r]++; ______r; })) ({ __might_sleep("include/crypto/scatterwalk.h", 53, 0); _cond_resched(); }); } static inline __attribute__((always_inline)) void scatterwalk_sg_chain(struct scatterlist *sg1, int num, struct scatterlist *sg2) { sg_set_page(&sg1[num - 1], (void *)sg2, 0, 0); sg1[num - 1].page_link &= ~0x02; } static inline __attribute__((always_inline)) struct scatterlist *scatterwalk_sg_next(struct scatterlist *sg) { if (__builtin_constant_p(((((sg)->page_link & 0x02)))) ? !!((((sg)->page_link & 0x02))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/crypto/scatterwalk.h", .line = 65, }; ______r = !!((((sg)->page_link & 0x02))); ______f.miss_hit[______r]++; ______r; })) return ((void *)0); return (++sg)->length ? sg : (void *)sg_page(sg); } static inline __attribute__((always_inline)) void scatterwalk_crypto_chain(struct scatterlist *head, struct scatterlist *sg, int chain, int num) { if (__builtin_constant_p(((chain))) ? !!((chain)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/crypto/scatterwalk.h", .line = 75, }; ______r = !!((chain)); ______f.miss_hit[______r]++; ______r; })) { head->length += sg->length; sg = scatterwalk_sg_next(sg); } if (__builtin_constant_p(((sg))) ? !!((sg)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/crypto/scatterwalk.h", .line = 80, }; ______r = !!((sg)); ______f.miss_hit[______r]++; ______r; })) scatterwalk_sg_chain(head, num, sg); else sg_mark_end(head); } static inline __attribute__((always_inline)) unsigned long scatterwalk_samebuf(struct scatter_walk *walk_in, struct scatter_walk *walk_out) { return !(((sg_page(walk_in->sg) - sg_page(walk_out->sg)) << 12) + (int)(walk_in->offset - walk_out->offset)); } static inline __attribute__((always_inline)) unsigned int scatterwalk_pagelen(struct scatter_walk *walk) { unsigned int len = walk->sg->offset + walk->sg->length - walk->offset; unsigned int len_this_page = ((unsigned long)(~walk->offset) & ~(~(((1UL) << 12)-1))) + 1; return len_this_page > len ? len : len_this_page; } static inline __attribute__((always_inline)) unsigned int scatterwalk_clamp(struct scatter_walk *walk, unsigned int nbytes) { unsigned int len_this_page = scatterwalk_pagelen(walk); return nbytes > len_this_page ? len_this_page : nbytes; } static inline __attribute__((always_inline)) void scatterwalk_advance(struct scatter_walk *walk, unsigned int nbytes) { walk->offset += nbytes; } static inline __attribute__((always_inline)) unsigned int scatterwalk_aligned(struct scatter_walk *walk, unsigned int alignmask) { return !(walk->offset & alignmask); } static inline __attribute__((always_inline)) struct page *scatterwalk_page(struct scatter_walk *walk) { return sg_page(walk->sg) + (walk->offset >> 12); } static inline __attribute__((always_inline)) void scatterwalk_unmap(void *vaddr, int out) { crypto_kunmap(vaddr, out); } void scatterwalk_start(struct scatter_walk *walk, struct scatterlist *sg); void scatterwalk_copychunks(void *buf, struct scatter_walk *walk, size_t nbytes, int out); void *scatterwalk_map(struct scatter_walk *walk, int out); void scatterwalk_done(struct scatter_walk *walk, int out, int more); void scatterwalk_map_and_copy(void *buf, struct scatterlist *sg, unsigned int start, unsigned int nbytes, int out); struct aead_givcrypt_request { u64 seq; u8 *giv; struct aead_request areq; }; static inline __attribute__((always_inline)) struct crypto_aead *aead_givcrypt_reqtfm( struct aead_givcrypt_request *req) { return crypto_aead_reqtfm(&req->areq); } static inline __attribute__((always_inline)) int crypto_aead_givencrypt(struct aead_givcrypt_request *req) { struct aead_tfm *crt = crypto_aead_crt(aead_givcrypt_reqtfm(req)); return crt->givencrypt(req); }; static inline __attribute__((always_inline)) int crypto_aead_givdecrypt(struct aead_givcrypt_request *req) { struct aead_tfm *crt = crypto_aead_crt(aead_givcrypt_reqtfm(req)); return crt->givdecrypt(req); }; static inline __attribute__((always_inline)) void aead_givcrypt_set_tfm(struct aead_givcrypt_request *req, struct crypto_aead *tfm) { req->areq.base.tfm = crypto_aead_tfm(tfm); } static inline __attribute__((always_inline)) struct aead_givcrypt_request *aead_givcrypt_alloc( struct crypto_aead *tfm, gfp_t gfp) { struct aead_givcrypt_request *req; req = kmalloc(sizeof(struct aead_givcrypt_request) + crypto_aead_reqsize(tfm), gfp); if (__builtin_constant_p((((__builtin_constant_p(req) ? !!(req) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/crypto/aead.h", .line = 65, }; ______r = __builtin_expect(!!(req), 1); ftrace_likely_update(&______f, ______r, 1); ______r; }))))) ? !!(((__builtin_constant_p(req) ? !!(req) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/crypto/aead.h", .line = 65, }; ______r = __builtin_expect(!!(req), 1); ftrace_likely_update(&______f, ______r, 1); ______r; })))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "include/crypto/aead.h", .line = 65, }; ______r = !!(((__builtin_constant_p(req) ? !!(req) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_annotated_branch"))) ______f = { .func = __func__, .file = "include/crypto/aead.h", .line = 65, }; ______r = __builtin_expect(!!(req), 1); ftrace_likely_update(&______f, ______r, 1); ______r; })))); ______f.miss_hit[______r]++; ______r; })) aead_givcrypt_set_tfm(req, tfm); return req; } static inline __attribute__((always_inline)) void aead_givcrypt_free(struct aead_givcrypt_request *req) { kfree(req); } static inline __attribute__((always_inline)) void aead_givcrypt_set_callback( struct aead_givcrypt_request *req, u32 flags, crypto_completion_t complete, void *data) { aead_request_set_callback(&req->areq, flags, complete, data); } static inline __attribute__((always_inline)) void aead_givcrypt_set_crypt(struct aead_givcrypt_request *req, struct scatterlist *src, struct scatterlist *dst, unsigned int nbytes, void *iv) { aead_request_set_crypt(&req->areq, src, dst, nbytes, iv); } static inline __attribute__((always_inline)) void aead_givcrypt_set_assoc(struct aead_givcrypt_request *req, struct scatterlist *assoc, unsigned int assoclen) { aead_request_set_assoc(&req->areq, assoc, assoclen); } static inline __attribute__((always_inline)) void aead_givcrypt_set_giv(struct aead_givcrypt_request *req, u8 *giv, u64 seq) { req->giv = giv; req->seq = seq; } struct rtattr; struct crypto_aead_spawn { struct crypto_spawn base; }; extern const struct crypto_type crypto_nivaead_type; static inline __attribute__((always_inline)) void crypto_set_aead_spawn( struct crypto_aead_spawn *spawn, struct crypto_instance *inst) { crypto_set_spawn(&spawn->base, inst); } int crypto_grab_aead(struct crypto_aead_spawn *spawn, const char *name, u32 type, u32 mask); static inline __attribute__((always_inline)) void crypto_drop_aead(struct crypto_aead_spawn *spawn) { crypto_drop_spawn(&spawn->base); } static inline __attribute__((always_inline)) struct crypto_alg *crypto_aead_spawn_alg( struct crypto_aead_spawn *spawn) { return spawn->base.alg; } static inline __attribute__((always_inline)) struct crypto_aead *crypto_spawn_aead( struct crypto_aead_spawn *spawn) { return __crypto_aead_cast( crypto_spawn_tfm(&spawn->base, 0x00000003, 0x0000000f)); } struct crypto_instance *aead_geniv_alloc(struct crypto_template *tmpl, struct rtattr **tb, u32 type, u32 mask); void aead_geniv_free(struct crypto_instance *inst); int aead_geniv_init(struct crypto_tfm *tfm); void aead_geniv_exit(struct crypto_tfm *tfm); static inline __attribute__((always_inline)) struct crypto_aead *aead_geniv_base(struct crypto_aead *geniv) { return crypto_aead_crt(geniv)->base; } static inline __attribute__((always_inline)) void *aead_givcrypt_reqctx(struct aead_givcrypt_request *req) { return aead_request_ctx(&req->areq); } static inline __attribute__((always_inline)) void aead_givcrypt_complete(struct aead_givcrypt_request *req, int err) { aead_request_complete(&req->areq, err); } struct async_aes_ctx { struct cryptd_ablkcipher *cryptd_tfm; }; struct aesni_rfc4106_gcm_ctx { u8 hash_subkey[16]; struct crypto_aes_ctx aes_key_expanded; u8 nonce[4]; struct cryptd_aead *cryptd_tfm; }; struct aesni_gcm_set_hash_subkey_result { int err; struct completion completion; }; struct aesni_hash_subkey_req_data { u8 iv[16]; struct aesni_gcm_set_hash_subkey_result result; struct scatterlist sg; }; __attribute__((regparm(0))) int aesni_set_key(struct crypto_aes_ctx *ctx, const u8 *in_key, unsigned int key_len); __attribute__((regparm(0))) void aesni_enc(struct crypto_aes_ctx *ctx, u8 *out, const u8 *in); __attribute__((regparm(0))) void aesni_dec(struct crypto_aes_ctx *ctx, u8 *out, const u8 *in); __attribute__((regparm(0))) void aesni_ecb_enc(struct crypto_aes_ctx *ctx, u8 *out, const u8 *in, unsigned int len); __attribute__((regparm(0))) void aesni_ecb_dec(struct crypto_aes_ctx *ctx, u8 *out, const u8 *in, unsigned int len); __attribute__((regparm(0))) void aesni_cbc_enc(struct crypto_aes_ctx *ctx, u8 *out, const u8 *in, unsigned int len, u8 *iv); __attribute__((regparm(0))) void aesni_cbc_dec(struct crypto_aes_ctx *ctx, u8 *out, const u8 *in, unsigned int len, u8 *iv); int crypto_fpu_init(void); void crypto_fpu_exit(void); static inline __attribute__((always_inline)) struct crypto_aes_ctx *aes_ctx(void *raw_ctx) { unsigned long addr = (unsigned long)raw_ctx; unsigned long align = (16); if (__builtin_constant_p(((align <= crypto_tfm_ctx_alignment()))) ? !!((align <= crypto_tfm_ctx_alignment())) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "arch/x86/crypto/aesni-intel_glue.c", .line = 162, }; ______r = !!((align <= crypto_tfm_ctx_alignment())); ______f.miss_hit[______r]++; ______r; })) align = 1; return (struct crypto_aes_ctx *)((((addr)) + ((typeof((addr)))((align)) - 1)) & ~((typeof((addr)))((align)) - 1)); } static int aes_set_key_common(struct crypto_tfm *tfm, void *raw_ctx, const u8 *in_key, unsigned int key_len) { struct crypto_aes_ctx *ctx = aes_ctx(raw_ctx); u32 *flags = &tfm->crt_flags; int err; if (__builtin_constant_p(((key_len != 16 && key_len != 24 && key_len != 32))) ? !!((key_len != 16 && key_len != 24 && key_len != 32)) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "arch/x86/crypto/aesni-intel_glue.c" , .line = 175 , }; ______r = !!((key_len != 16 && key_len != 24 && key_len != 32)); ______f.miss_hit[______r]++; ______r; })) { *flags |= 0x00200000; return -22; } if (__builtin_constant_p(((!irq_fpu_usable()))) ? !!((!irq_fpu_usable())) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "arch/x86/crypto/aesni-intel_glue.c", .line = 180, }; ______r = !!((!irq_fpu_usable())); ______f.miss_hit[______r]++; ______r; })) err = crypto_aes_expand_key(ctx, in_key, key_len); else { kernel_fpu_begin(); err = aesni_set_key(ctx, in_key, key_len); kernel_fpu_end(); } return err; } static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key, unsigned int key_len) { return aes_set_key_common(tfm, crypto_tfm_ctx(tfm), in_key, key_len); } static void aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) { struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); if (__builtin_constant_p(((!irq_fpu_usable()))) ? !!((!irq_fpu_usable())) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "arch/x86/crypto/aesni-intel_glue.c", .line = 201, }; ______r = !!((!irq_fpu_usable())); ______f.miss_hit[______r]++; ______r; })) crypto_aes_encrypt_x86(ctx, dst, src); else { kernel_fpu_begin(); aesni_enc(ctx, dst, src); kernel_fpu_end(); } } static void aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) { struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); if (__builtin_constant_p(((!irq_fpu_usable()))) ? !!((!irq_fpu_usable())) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "arch/x86/crypto/aesni-intel_glue.c", .line = 214, }; ______r = !!((!irq_fpu_usable())); ______f.miss_hit[______r]++; ______r; })) crypto_aes_decrypt_x86(ctx, dst, src); else { kernel_fpu_begin(); aesni_dec(ctx, dst, src); kernel_fpu_end(); } } static struct crypto_alg aesni_alg = { .cra_name = "aes", .cra_driver_name = "aes-aesni", .cra_priority = 300, .cra_flags = 0x00000001, .cra_blocksize = 16, .cra_ctxsize = sizeof(struct crypto_aes_ctx)+(16)-1, .cra_alignmask = 0, .cra_module = (&__this_module), .cra_list = { &(aesni_alg.cra_list), &(aesni_alg.cra_list) }, .cra_u = { .cipher = { .cia_min_keysize = 16, .cia_max_keysize = 32, .cia_setkey = aes_set_key, .cia_encrypt = aes_encrypt, .cia_decrypt = aes_decrypt } } }; static void __aes_encrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) { struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); aesni_enc(ctx, dst, src); } static void __aes_decrypt(struct crypto_tfm *tfm, u8 *dst, const u8 *src) { struct crypto_aes_ctx *ctx = aes_ctx(crypto_tfm_ctx(tfm)); aesni_dec(ctx, dst, src); } static struct crypto_alg __aesni_alg = { .cra_name = "__aes-aesni", .cra_driver_name = "__driver-aes-aesni", .cra_priority = 0, .cra_flags = 0x00000001, .cra_blocksize = 16, .cra_ctxsize = sizeof(struct crypto_aes_ctx)+(16)-1, .cra_alignmask = 0, .cra_module = (&__this_module), .cra_list = { &(__aesni_alg.cra_list), &(__aesni_alg.cra_list) }, .cra_u = { .cipher = { .cia_min_keysize = 16, .cia_max_keysize = 32, .cia_setkey = aes_set_key, .cia_encrypt = __aes_encrypt, .cia_decrypt = __aes_decrypt } } }; static int ecb_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); struct blkcipher_walk walk; int err; blkcipher_walk_init(&walk, dst, src, nbytes); err = blkcipher_walk_virt(desc, &walk); desc->flags &= ~0x00000200; kernel_fpu_begin(); while ((nbytes = walk.nbytes)) { aesni_ecb_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, nbytes & (~(16 -1))); nbytes &= 16 - 1; err = blkcipher_walk_done(desc, &walk, nbytes); } kernel_fpu_end(); return err; } static int ecb_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); struct blkcipher_walk walk; int err; blkcipher_walk_init(&walk, dst, src, nbytes); err = blkcipher_walk_virt(desc, &walk); desc->flags &= ~0x00000200; kernel_fpu_begin(); while ((nbytes = walk.nbytes)) { aesni_ecb_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, nbytes & (~(16 -1))); nbytes &= 16 - 1; err = blkcipher_walk_done(desc, &walk, nbytes); } kernel_fpu_end(); return err; } static struct crypto_alg blk_ecb_alg = { .cra_name = "__ecb-aes-aesni", .cra_driver_name = "__driver-ecb-aes-aesni", .cra_priority = 0, .cra_flags = 0x00000004, .cra_blocksize = 16, .cra_ctxsize = sizeof(struct crypto_aes_ctx)+(16)-1, .cra_alignmask = 0, .cra_type = &crypto_blkcipher_type, .cra_module = (&__this_module), .cra_list = { &(blk_ecb_alg.cra_list), &(blk_ecb_alg.cra_list) }, .cra_u = { .blkcipher = { .min_keysize = 16, .max_keysize = 32, .setkey = aes_set_key, .encrypt = ecb_encrypt, .decrypt = ecb_decrypt, }, }, }; static int cbc_encrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); struct blkcipher_walk walk; int err; blkcipher_walk_init(&walk, dst, src, nbytes); err = blkcipher_walk_virt(desc, &walk); desc->flags &= ~0x00000200; kernel_fpu_begin(); while ((nbytes = walk.nbytes)) { aesni_cbc_enc(ctx, walk.dst.virt.addr, walk.src.virt.addr, nbytes & (~(16 -1)), walk.iv); nbytes &= 16 - 1; err = blkcipher_walk_done(desc, &walk, nbytes); } kernel_fpu_end(); return err; } static int cbc_decrypt(struct blkcipher_desc *desc, struct scatterlist *dst, struct scatterlist *src, unsigned int nbytes) { struct crypto_aes_ctx *ctx = aes_ctx(crypto_blkcipher_ctx(desc->tfm)); struct blkcipher_walk walk; int err; blkcipher_walk_init(&walk, dst, src, nbytes); err = blkcipher_walk_virt(desc, &walk); desc->flags &= ~0x00000200; kernel_fpu_begin(); while ((nbytes = walk.nbytes)) { aesni_cbc_dec(ctx, walk.dst.virt.addr, walk.src.virt.addr, nbytes & (~(16 -1)), walk.iv); nbytes &= 16 - 1; err = blkcipher_walk_done(desc, &walk, nbytes); } kernel_fpu_end(); return err; } static struct crypto_alg blk_cbc_alg = { .cra_name = "__cbc-aes-aesni", .cra_driver_name = "__driver-cbc-aes-aesni", .cra_priority = 0, .cra_flags = 0x00000004, .cra_blocksize = 16, .cra_ctxsize = sizeof(struct crypto_aes_ctx)+(16)-1, .cra_alignmask = 0, .cra_type = &crypto_blkcipher_type, .cra_module = (&__this_module), .cra_list = { &(blk_cbc_alg.cra_list), &(blk_cbc_alg.cra_list) }, .cra_u = { .blkcipher = { .min_keysize = 16, .max_keysize = 32, .setkey = aes_set_key, .encrypt = cbc_encrypt, .decrypt = cbc_decrypt, }, }, }; static int ablk_set_key(struct crypto_ablkcipher *tfm, const u8 *key, unsigned int key_len) { struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm); struct crypto_ablkcipher *child = &ctx->cryptd_tfm->base; int err; crypto_ablkcipher_clear_flags(child, 0x000fff00); crypto_ablkcipher_set_flags(child, crypto_ablkcipher_get_flags(tfm) & 0x000fff00); err = crypto_ablkcipher_setkey(child, key, key_len); crypto_ablkcipher_set_flags(tfm, crypto_ablkcipher_get_flags(child) & 0xfff00000); return err; } static int ablk_encrypt(struct ablkcipher_request *req) { struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm); if (__builtin_constant_p(((!irq_fpu_usable()))) ? !!((!irq_fpu_usable())) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "arch/x86/crypto/aesni-intel_glue.c", .line = 508, }; ______r = !!((!irq_fpu_usable())); ______f.miss_hit[______r]++; ______r; })) { struct ablkcipher_request *cryptd_req = ablkcipher_request_ctx(req); __builtin_memcpy(cryptd_req, req, sizeof(*req)); ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); return crypto_ablkcipher_encrypt(cryptd_req); } else { struct blkcipher_desc desc; desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm); desc.info = req->info; desc.flags = 0; return crypto_blkcipher_crt(desc.tfm)->encrypt( &desc, req->dst, req->src, req->nbytes); } } static int ablk_decrypt(struct ablkcipher_request *req) { struct crypto_ablkcipher *tfm = crypto_ablkcipher_reqtfm(req); struct async_aes_ctx *ctx = crypto_ablkcipher_ctx(tfm); if (__builtin_constant_p(((!irq_fpu_usable()))) ? !!((!irq_fpu_usable())) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "arch/x86/crypto/aesni-intel_glue.c", .line = 529, }; ______r = !!((!irq_fpu_usable())); ______f.miss_hit[______r]++; ______r; })) { struct ablkcipher_request *cryptd_req = ablkcipher_request_ctx(req); __builtin_memcpy(cryptd_req, req, sizeof(*req)); ablkcipher_request_set_tfm(cryptd_req, &ctx->cryptd_tfm->base); return crypto_ablkcipher_decrypt(cryptd_req); } else { struct blkcipher_desc desc; desc.tfm = cryptd_ablkcipher_child(ctx->cryptd_tfm); desc.info = req->info; desc.flags = 0; return crypto_blkcipher_crt(desc.tfm)->decrypt( &desc, req->dst, req->src, req->nbytes); } } static void ablk_exit(struct crypto_tfm *tfm) { struct async_aes_ctx *ctx = crypto_tfm_ctx(tfm); cryptd_free_ablkcipher(ctx->cryptd_tfm); } static void ablk_init_common(struct crypto_tfm *tfm, struct cryptd_ablkcipher *cryptd_tfm) { struct async_aes_ctx *ctx = crypto_tfm_ctx(tfm); ctx->cryptd_tfm = cryptd_tfm; tfm->crt_u.ablkcipher.reqsize = sizeof(struct ablkcipher_request) + crypto_ablkcipher_reqsize(&cryptd_tfm->base); } static int ablk_ecb_init(struct crypto_tfm *tfm) { struct cryptd_ablkcipher *cryptd_tfm; cryptd_tfm = cryptd_alloc_ablkcipher("__driver-ecb-aes-aesni", 0, 0); if (__builtin_constant_p(((IS_ERR(cryptd_tfm)))) ? !!((IS_ERR(cryptd_tfm))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "arch/x86/crypto/aesni-intel_glue.c", .line = 567, }; ______r = !!((IS_ERR(cryptd_tfm))); ______f.miss_hit[______r]++; ______r; })) return PTR_ERR(cryptd_tfm); ablk_init_common(tfm, cryptd_tfm); return 0; } static struct crypto_alg ablk_ecb_alg = { .cra_name = "ecb(aes)", .cra_driver_name = "ecb-aes-aesni", .cra_priority = 400, .cra_flags = 0x00000005|0x00000080, .cra_blocksize = 16, .cra_ctxsize = sizeof(struct async_aes_ctx), .cra_alignmask = 0, .cra_type = &crypto_ablkcipher_type, .cra_module = (&__this_module), .cra_list = { &(ablk_ecb_alg.cra_list), &(ablk_ecb_alg.cra_list) }, .cra_init = ablk_ecb_init, .cra_exit = ablk_exit, .cra_u = { .ablkcipher = { .min_keysize = 16, .max_keysize = 32, .setkey = ablk_set_key, .encrypt = ablk_encrypt, .decrypt = ablk_decrypt, }, }, }; static int ablk_cbc_init(struct crypto_tfm *tfm) { struct cryptd_ablkcipher *cryptd_tfm; cryptd_tfm = cryptd_alloc_ablkcipher("__driver-cbc-aes-aesni", 0, 0); if (__builtin_constant_p(((IS_ERR(cryptd_tfm)))) ? !!((IS_ERR(cryptd_tfm))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "arch/x86/crypto/aesni-intel_glue.c", .line = 602, }; ______r = !!((IS_ERR(cryptd_tfm))); ______f.miss_hit[______r]++; ______r; })) return PTR_ERR(cryptd_tfm); ablk_init_common(tfm, cryptd_tfm); return 0; } static struct crypto_alg ablk_cbc_alg = { .cra_name = "cbc(aes)", .cra_driver_name = "cbc-aes-aesni", .cra_priority = 400, .cra_flags = 0x00000005|0x00000080, .cra_blocksize = 16, .cra_ctxsize = sizeof(struct async_aes_ctx), .cra_alignmask = 0, .cra_type = &crypto_ablkcipher_type, .cra_module = (&__this_module), .cra_list = { &(ablk_cbc_alg.cra_list), &(ablk_cbc_alg.cra_list) }, .cra_init = ablk_cbc_init, .cra_exit = ablk_exit, .cra_u = { .ablkcipher = { .min_keysize = 16, .max_keysize = 32, .ivsize = 16, .setkey = ablk_set_key, .encrypt = ablk_encrypt, .decrypt = ablk_decrypt, }, }, }; static int ablk_pcbc_init(struct crypto_tfm *tfm) { struct cryptd_ablkcipher *cryptd_tfm; cryptd_tfm = cryptd_alloc_ablkcipher("fpu(pcbc(__driver-aes-aesni))", 0, 0); if (__builtin_constant_p(((IS_ERR(cryptd_tfm)))) ? !!((IS_ERR(cryptd_tfm))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "arch/x86/crypto/aesni-intel_glue.c", .line = 758, }; ______r = !!((IS_ERR(cryptd_tfm))); ______f.miss_hit[______r]++; ______r; })) return PTR_ERR(cryptd_tfm); ablk_init_common(tfm, cryptd_tfm); return 0; } static struct crypto_alg ablk_pcbc_alg = { .cra_name = "pcbc(aes)", .cra_driver_name = "pcbc-aes-aesni", .cra_priority = 400, .cra_flags = 0x00000005|0x00000080, .cra_blocksize = 16, .cra_ctxsize = sizeof(struct async_aes_ctx), .cra_alignmask = 0, .cra_type = &crypto_ablkcipher_type, .cra_module = (&__this_module), .cra_list = { &(ablk_pcbc_alg.cra_list), &(ablk_pcbc_alg.cra_list) }, .cra_init = ablk_pcbc_init, .cra_exit = ablk_exit, .cra_u = { .ablkcipher = { .min_keysize = 16, .max_keysize = 32, .ivsize = 16, .setkey = ablk_set_key, .encrypt = ablk_encrypt, .decrypt = ablk_decrypt, }, }, }; static int __attribute__ ((__section__(".init.text"))) __attribute__((__cold__)) __attribute__((no_instrument_function)) aesni_init(void) { int err; if (__builtin_constant_p(((!(__builtin_constant_p((4*32+25)) && ( ((((4*32+25))>>5)==0 && (1UL<<(((4*32+25))&31) & ((1<<((0*32+ 0) & 31))|0|0|(1<<((0*32+ 6) & 31))| (1<<((0*32+ 8) & 31))|0|0|(1<<((0*32+15) & 31))| 0|0))) || ((((4*32+25))>>5)==1 && (1UL<<(((4*32+25))&31) & (0|0))) || ((((4*32+25))>>5)==2 && (1UL<<(((4*32+25))&31) & 0)) || ((((4*32+25))>>5)==3 && (1UL<<(((4*32+25))&31) & (0))) || ((((4*32+25))>>5)==4 && (1UL<<(((4*32+25))&31) & 0)) || ((((4*32+25))>>5)==5 && (1UL<<(((4*32+25))&31) & 0)) || ((((4*32+25))>>5)==6 && (1UL<<(((4*32+25))&31) & 0)) || ((((4*32+25))>>5)==7 && (1UL<<(((4*32+25))&31) & 0)) || ((((4*32+25))>>5)==8 && (1UL<<(((4*32+25))&31) & 0)) || ((((4*32+25))>>5)==9 && (1UL<<(((4*32+25))&31) & 0)) ) ? 1 : (__builtin_constant_p(((4*32+25))) ? constant_test_bit(((4*32+25)), ((unsigned long *)((&boot_cpu_data)->x86_capability))) : variable_test_bit(((4*32+25)), ((unsigned long *)((&boot_cpu_data)->x86_capability)))))))) ? !!((!(__builtin_constant_p((4*32+25)) && ( ((((4*32+25))>>5)==0 && (1UL<<(((4*32+25))&31) & ((1<<((0*32+ 0) & 31))|0|0|(1<<((0*32+ 6) & 31))| (1<<((0*32+ 8) & 31))|0|0|(1<<((0*32+15) & 31))| 0|0))) || ((((4*32+25))>>5)==1 && (1UL<<(((4*32+25))&31) & (0|0))) || ((((4*32+25))>>5)==2 && (1UL<<(((4*32+25))&31) & 0)) || ((((4*32+25))>>5)==3 && (1UL<<(((4*32+25))&31) & (0))) || ((((4*32+25))>>5)==4 && (1UL<<(((4*32+25))&31) & 0)) || ((((4*32+25))>>5)==5 && (1UL<<(((4*32+25))&31) & 0)) || ((((4*32+25))>>5)==6 && (1UL<<(((4*32+25))&31) & 0)) || ((((4*32+25))>>5)==7 && (1UL<<(((4*32+25))&31) & 0)) || ((((4*32+25))>>5)==8 && (1UL<<(((4*32+25))&31) & 0)) || ((((4*32+25))>>5)==9 && (1UL<<(((4*32+25))&31) & 0)) ) ? 1 : (__builtin_constant_p(((4*32+25))) ? constant_test_bit(((4*32+25)), ((unsigned long *)((&boot_cpu_data)->x86_capability))) : variable_test_bit(((4*32+25)), ((unsigned long *)((&boot_cpu_data)->x86_capability))))))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "arch/x86/crypto/aesni-intel_glue.c", .line = 1259, }; ______r = !!((!(__builtin_constant_p((4*32+25)) && ( ((((4*32+25))>>5)==0 && (1UL<<(((4*32+25))&31) & ((1<<((0*32+ 0) & 31))|0|0|(1<<((0*32+ 6) & 31))| (1<<((0*32+ 8) & 31))|0|0|(1<<((0*32+15) & 31))| 0|0))) || ((((4*32+25))>>5)==1 && (1UL<<(((4*32+25))&31) & (0|0))) || ((((4*32+25))>>5)==2 && (1UL<<(((4*32+25))&31) & 0)) || ((((4*32+25))>>5)==3 && (1UL<<(((4*32+25))&31) & (0))) || ((((4*32+25))>>5)==4 && (1UL<<(((4*32+25))&31) & 0)) || ((((4*32+25))>>5)==5 && (1UL<<(((4*32+25))&31) & 0)) || ((((4*32+25))>>5)==6 && (1UL<<(((4*32+25))&31) & 0)) || ((((4*32+25))>>5)==7 && (1UL<<(((4*32+25))&31) & 0)) || ((((4*32+25))>>5)==8 && (1UL<<(((4*32+25))&31) & 0)) || ((((4*32+25))>>5)==9 && (1UL<<(((4*32+25))&31) & 0)) ) ? 1 : (__builtin_constant_p(((4*32+25))) ? constant_test_bit(((4*32+25)), ((unsigned long *)((&boot_cpu_data)->x86_capability))) : variable_test_bit(((4*32+25)), ((unsigned long *)((&boot_cpu_data)->x86_capability))))))); ______f.miss_hit[______r]++; ______r; })) { printk("<6>" "Intel AES-NI instructions are not detected.\n"); return -19; } if (__builtin_constant_p((((err = crypto_fpu_init())))) ? !!(((err = crypto_fpu_init()))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "arch/x86/crypto/aesni-intel_glue.c", .line = 1264, }; ______r = !!(((err = crypto_fpu_init()))); ______f.miss_hit[______r]++; ______r; })) goto fpu_err; if (__builtin_constant_p((((err = crypto_register_alg(&aesni_alg))))) ? !!(((err = crypto_register_alg(&aesni_alg)))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "arch/x86/crypto/aesni-intel_glue.c", .line = 1266, }; ______r = !!(((err = crypto_register_alg(&aesni_alg)))); ______f.miss_hit[______r]++; ______r; })) goto aes_err; if (__builtin_constant_p((((err = crypto_register_alg(&__aesni_alg))))) ? !!(((err = crypto_register_alg(&__aesni_alg)))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "arch/x86/crypto/aesni-intel_glue.c", .line = 1268, }; ______r = !!(((err = crypto_register_alg(&__aesni_alg)))); ______f.miss_hit[______r]++; ______r; })) goto __aes_err; if (__builtin_constant_p((((err = crypto_register_alg(&blk_ecb_alg))))) ? !!(((err = crypto_register_alg(&blk_ecb_alg)))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "arch/x86/crypto/aesni-intel_glue.c", .line = 1270, }; ______r = !!(((err = crypto_register_alg(&blk_ecb_alg)))); ______f.miss_hit[______r]++; ______r; })) goto blk_ecb_err; if (__builtin_constant_p((((err = crypto_register_alg(&blk_cbc_alg))))) ? !!(((err = crypto_register_alg(&blk_cbc_alg)))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "arch/x86/crypto/aesni-intel_glue.c", .line = 1272, }; ______r = !!(((err = crypto_register_alg(&blk_cbc_alg)))); ______f.miss_hit[______r]++; ______r; })) goto blk_cbc_err; if (__builtin_constant_p((((err = crypto_register_alg(&ablk_ecb_alg))))) ? !!(((err = crypto_register_alg(&ablk_ecb_alg)))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "arch/x86/crypto/aesni-intel_glue.c", .line = 1274, }; ______r = !!(((err = crypto_register_alg(&ablk_ecb_alg)))); ______f.miss_hit[______r]++; ______r; })) goto ablk_ecb_err; if (__builtin_constant_p((((err = crypto_register_alg(&ablk_cbc_alg))))) ? !!(((err = crypto_register_alg(&ablk_cbc_alg)))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "arch/x86/crypto/aesni-intel_glue.c", .line = 1276, }; ______r = !!(((err = crypto_register_alg(&ablk_cbc_alg)))); ______f.miss_hit[______r]++; ______r; })) goto ablk_cbc_err; if (__builtin_constant_p((((err = crypto_register_alg(&ablk_pcbc_alg))))) ? !!(((err = crypto_register_alg(&ablk_pcbc_alg)))) : ({ int ______r; static struct ftrace_branch_data __attribute__((__aligned__(4))) __attribute__((section("_ftrace_branch"))) ______f = { .func = __func__, .file = "arch/x86/crypto/aesni-intel_glue.c", .line = 1297, }; ______r = !!(((err = crypto_register_alg(&ablk_pcbc_alg)))); ______f.miss_hit[______r]++; ______r; })) goto ablk_pcbc_err; return err; crypto_unregister_alg(&ablk_pcbc_alg); ablk_pcbc_err: crypto_unregister_alg(&ablk_cbc_alg); ablk_cbc_err: crypto_unregister_alg(&ablk_ecb_alg); ablk_ecb_err: crypto_unregister_alg(&blk_cbc_alg); blk_cbc_err: crypto_unregister_alg(&blk_ecb_alg); blk_ecb_err: crypto_unregister_alg(&__aesni_alg); __aes_err: crypto_unregister_alg(&aesni_alg); aes_err: fpu_err: return err; } static void __attribute__ ((__section__(".exit.text"))) __attribute__((__cold__)) __attribute__((no_instrument_function)) aesni_exit(void) { crypto_unregister_alg(&ablk_pcbc_alg); crypto_unregister_alg(&ablk_cbc_alg); crypto_unregister_alg(&ablk_ecb_alg); crypto_unregister_alg(&blk_cbc_alg); crypto_unregister_alg(&blk_ecb_alg); crypto_unregister_alg(&__aesni_alg); crypto_unregister_alg(&aesni_alg); crypto_fpu_exit(); } static inline __attribute__((always_inline)) initcall_t __inittest(void) { return aesni_init; } int init_module(void) __attribute__((alias("aesni_init")));; static inline __attribute__((always_inline)) exitcall_t __exittest(void) { return aesni_exit; } void cleanup_module(void) __attribute__((alias("aesni_exit")));; static const char __mod_description1380[] __attribute__((__used__)) __attribute__((section(".modinfo"), unused, aligned(1))) = "description" "=" "Rijndael (AES) Cipher Algorithm, Intel AES-NI instructions optimized"; static const char __mod_license1381[] __attribute__((__used__)) __attribute__((section(".modinfo"), unused, aligned(1))) = "license" "=" "GPL"; static const char __mod_alias1382[] __attribute__((__used__)) __attribute__((section(".modinfo"), unused, aligned(1))) = "alias" "=" "aes";