#ifndef __TOOLS_LINUX_PERCPU_H #define __TOOLS_LINUX_PERCPU_H #include #define __percpu #define free_percpu(percpu) free(percpu) #define __alloc_percpu_gfp(size, align, gfp) calloc(1, size) #define __alloc_percpu(size, align) calloc(1, size) #define alloc_percpu_gfp(type, gfp) \ (typeof(type) __percpu *)__alloc_percpu_gfp(sizeof(type), \ __alignof__(type), gfp) #define alloc_percpu(type) \ (typeof(type) __percpu *)__alloc_percpu(sizeof(type), \ __alignof__(type)) #define __verify_pcpu_ptr(ptr) #define per_cpu_ptr(ptr, cpu) (ptr) #define raw_cpu_ptr(ptr) (ptr) #define this_cpu_ptr(ptr) raw_cpu_ptr(ptr) #define __pcpu_size_call_return(stem, variable) \ ({ \ typeof(variable) pscr_ret__; \ __verify_pcpu_ptr(&(variable)); \ switch(sizeof(variable)) { \ case 1: pscr_ret__ = stem##1(variable); break; \ case 2: pscr_ret__ = stem##2(variable); break; \ case 4: pscr_ret__ = stem##4(variable); break; \ case 8: pscr_ret__ = stem##8(variable); break; \ default: \ __bad_size_call_parameter(); break; \ } \ pscr_ret__; \ }) #define __pcpu_size_call_return2(stem, variable, ...) \ ({ \ typeof(variable) pscr2_ret__; \ __verify_pcpu_ptr(&(variable)); \ switch(sizeof(variable)) { \ case 1: pscr2_ret__ = stem##1(variable, __VA_ARGS__); break; \ case 2: pscr2_ret__ = stem##2(variable, __VA_ARGS__); break; \ case 4: pscr2_ret__ = stem##4(variable, __VA_ARGS__); break; \ case 8: pscr2_ret__ = stem##8(variable, __VA_ARGS__); break; \ default: \ __bad_size_call_parameter(); break; \ } \ pscr2_ret__; \ }) /* * Special handling for cmpxchg_double. cmpxchg_double is passed two * percpu variables. The first has to be aligned to a double word * boundary and the second has to follow directly thereafter. * We enforce this on all architectures even if they don't support * a double cmpxchg instruction, since it's a cheap requirement, and it * avoids breaking the requirement for architectures with the instruction. */ #define __pcpu_double_call_return_bool(stem, pcp1, pcp2, ...) \ ({ \ bool pdcrb_ret__; \ __verify_pcpu_ptr(&(pcp1)); \ BUILD_BUG_ON(sizeof(pcp1) != sizeof(pcp2)); \ VM_BUG_ON((unsigned long)(&(pcp1)) % (2 * sizeof(pcp1))); \ VM_BUG_ON((unsigned long)(&(pcp2)) != \ (unsigned long)(&(pcp1)) + sizeof(pcp1)); \ switch(sizeof(pcp1)) { \ case 1: pdcrb_ret__ = stem##1(pcp1, pcp2, __VA_ARGS__); break; \ case 2: pdcrb_ret__ = stem##2(pcp1, pcp2, __VA_ARGS__); break; \ case 4: pdcrb_ret__ = stem##4(pcp1, pcp2, __VA_ARGS__); break; \ case 8: pdcrb_ret__ = stem##8(pcp1, pcp2, __VA_ARGS__); break; \ default: \ __bad_size_call_parameter(); break; \ } \ pdcrb_ret__; \ }) #define __pcpu_size_call(stem, variable, ...) \ do { \ __verify_pcpu_ptr(&(variable)); \ switch(sizeof(variable)) { \ case 1: stem##1(variable, __VA_ARGS__);break; \ case 2: stem##2(variable, __VA_ARGS__);break; \ case 4: stem##4(variable, __VA_ARGS__);break; \ case 8: stem##8(variable, __VA_ARGS__);break; \ default: \ __bad_size_call_parameter();break; \ } \ } while (0) #define raw_cpu_read(pcp) __pcpu_size_call_return(raw_cpu_read_, pcp) #define raw_cpu_write(pcp, val) __pcpu_size_call(raw_cpu_write_, pcp, val) #define raw_cpu_add(pcp, val) __pcpu_size_call(raw_cpu_add_, pcp, val) #define raw_cpu_and(pcp, val) __pcpu_size_call(raw_cpu_and_, pcp, val) #define raw_cpu_or(pcp, val) __pcpu_size_call(raw_cpu_or_, pcp, val) #define raw_cpu_add_return(pcp, val) __pcpu_size_call_return2(raw_cpu_add_return_, pcp, val) #define raw_cpu_xchg(pcp, nval) __pcpu_size_call_return2(raw_cpu_xchg_, pcp, nval) #define raw_cpu_cmpxchg(pcp, oval, nval) \ __pcpu_size_call_return2(raw_cpu_cmpxchg_, pcp, oval, nval) #define raw_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \ __pcpu_double_call_return_bool(raw_cpu_cmpxchg_double_, pcp1, pcp2, oval1, oval2, nval1, nval2) #define raw_cpu_sub(pcp, val) raw_cpu_add(pcp, -(val)) #define raw_cpu_inc(pcp) raw_cpu_add(pcp, 1) #define raw_cpu_dec(pcp) raw_cpu_sub(pcp, 1) #define raw_cpu_sub_return(pcp, val) raw_cpu_add_return(pcp, -(typeof(pcp))(val)) #define raw_cpu_inc_return(pcp) raw_cpu_add_return(pcp, 1) #define raw_cpu_dec_return(pcp) raw_cpu_add_return(pcp, -1) #define __this_cpu_read(pcp) \ ({ \ raw_cpu_read(pcp); \ }) #define __this_cpu_write(pcp, val) \ ({ \ raw_cpu_write(pcp, val); \ }) #define __this_cpu_add(pcp, val) \ ({ \ raw_cpu_add(pcp, val); \ }) #define __this_cpu_and(pcp, val) \ ({ \ raw_cpu_and(pcp, val); \ }) #define __this_cpu_or(pcp, val) \ ({ \ raw_cpu_or(pcp, val); \ }) #define __this_cpu_add_return(pcp, val) \ ({ \ raw_cpu_add_return(pcp, val); \ }) #define __this_cpu_xchg(pcp, nval) \ ({ \ raw_cpu_xchg(pcp, nval); \ }) #define __this_cpu_cmpxchg(pcp, oval, nval) \ ({ \ raw_cpu_cmpxchg(pcp, oval, nval); \ }) #define __this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \ raw_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2); \ }) #define __this_cpu_sub(pcp, val) __this_cpu_add(pcp, -(typeof(pcp))(val)) #define __this_cpu_inc(pcp) __this_cpu_add(pcp, 1) #define __this_cpu_dec(pcp) __this_cpu_sub(pcp, 1) #define __this_cpu_sub_return(pcp, val) __this_cpu_add_return(pcp, -(typeof(pcp))(val)) #define __this_cpu_inc_return(pcp) __this_cpu_add_return(pcp, 1) #define __this_cpu_dec_return(pcp) __this_cpu_add_return(pcp, -1) #define this_cpu_read(pcp) ((pcp)) #define this_cpu_write(pcp, val) ((pcp) = val) #define this_cpu_add(pcp, val) ((pcp) += val) #define this_cpu_and(pcp, val) ((pcp) &= val) #define this_cpu_or(pcp, val) ((pcp) |= val) #define this_cpu_add_return(pcp, val) ((pcp) += val) #define this_cpu_xchg(pcp, nval) \ ({ \ typeof(pcp) _r = (pcp); \ (pcp) = (nval); \ _r; \ }) #define this_cpu_cmpxchg(pcp, oval, nval) \ __pcpu_size_call_return2(this_cpu_cmpxchg_, pcp, oval, nval) #define this_cpu_cmpxchg_double(pcp1, pcp2, oval1, oval2, nval1, nval2) \ __pcpu_double_call_return_bool(this_cpu_cmpxchg_double_, pcp1, pcp2, oval1, oval2, nval1, nval2) #define this_cpu_sub(pcp, val) this_cpu_add(pcp, -(typeof(pcp))(val)) #define this_cpu_inc(pcp) this_cpu_add(pcp, 1) #define this_cpu_dec(pcp) this_cpu_sub(pcp, 1) #define this_cpu_sub_return(pcp, val) this_cpu_add_return(pcp, -(typeof(pcp))(val)) #define this_cpu_inc_return(pcp) this_cpu_add_return(pcp, 1) #define this_cpu_dec_return(pcp) this_cpu_add_return(pcp, -1) #endif /* __TOOLS_LINUX_PERCPU_H */