diff options
| author | Stephen Rothwell <sfr@canb.auug.org.au> | 2008-05-07 13:52:16 +1000 |
|---|---|---|
| committer | Stephen Rothwell <sfr@canb.auug.org.au> | 2008-05-07 13:52:16 +1000 |
| commit | bcb1b8c75e30e9c39b6f162cffe59df9caaf27e4 (patch) | |
| tree | 4ab3f05bc3ef51b1296ca7a00ef7acdb7428e6f6 | |
| parent | f3636bd721198538840ec33389001280c3bd7452 (diff) | |
| parent | 553e3cc6fd94d3924617f1b36614606f12211219 (diff) | |
Merge commit 'sched-latest/latest'
Conflicts:
include/linux/stop_machine.h
kernel/stop_machine.c
185 files changed, 5421 insertions, 1470 deletions
diff --git a/Documentation/RCU/NMI-RCU.txt b/Documentation/RCU/NMI-RCU.txt index c64158ecde43..a6d32e65d222 100644 --- a/Documentation/RCU/NMI-RCU.txt +++ b/Documentation/RCU/NMI-RCU.txt @@ -93,6 +93,9 @@ Since NMI handlers disable preemption, synchronize_sched() is guaranteed not to return until all ongoing NMI handlers exit. It is therefore safe to free up the handler's data as soon as synchronize_sched() returns. +Important note: for this to work, the architecture in question must +invoke irq_enter() and irq_exit() on NMI entry and exit, respectively. + Answer to Quick Quiz diff --git a/Documentation/RCU/RTFP.txt b/Documentation/RCU/RTFP.txt index 39ad8f56783a..9f711d2df91b 100644 --- a/Documentation/RCU/RTFP.txt +++ b/Documentation/RCU/RTFP.txt @@ -52,6 +52,10 @@ of each iteration. Unfortunately, chaotic relaxation requires highly structured data, such as the matrices used in scientific programs, and is thus inapplicable to most data structures in operating-system kernels. +In 1992, Henry (now Alexia) Massalin completed a dissertation advising +parallel programmers to defer processing when feasible to simplify +synchronization. RCU makes extremely heavy use of this advice. + In 1993, Jacobson [Jacobson93] verbally described what is perhaps the simplest deferred-free technique: simply waiting a fixed amount of time before freeing blocks awaiting deferred free. Jacobson did not describe @@ -138,6 +142,13 @@ blocking in read-side critical sections appeared [PaulEMcKenney2006c], Robert Olsson described an RCU-protected trie-hash combination [RobertOlsson2006a]. +2007 saw the journal version of the award-winning RCU paper from 2006 +[ThomasEHart2007a], as well as a paper demonstrating use of Promela +and Spin to mechanically verify an optimization to Oleg Nesterov's +QRCU [PaulEMcKenney2007QRCUspin], a design document describing +preemptible RCU [PaulEMcKenney2007PreemptibleRCU], and the three-part +LWN "What is RCU?" series [PaulEMcKenney2007WhatIsRCUFundamentally, +PaulEMcKenney2008WhatIsRCUUsage, and PaulEMcKenney2008WhatIsRCUAPI]. Bibtex Entries @@ -202,6 +213,20 @@ Bibtex Entries ,Year="1991" } +@phdthesis{HMassalinPhD +,author="H. Massalin" +,title="Synthesis: An Efficient Implementation of Fundamental Operating +System Services" +,school="Columbia University" +,address="New York, NY" +,year="1992" +,annotation=" + Mondo optimizing compiler. + Wait-free stuff. + Good advice: defer work to avoid synchronization. +" +} + @unpublished{Jacobson93 ,author="Van Jacobson" ,title="Avoid Read-Side Locking Via Delayed Free" @@ -635,3 +660,86 @@ Revised: " } +@unpublished{PaulEMcKenney2007PreemptibleRCU +,Author="Paul E. McKenney" +,Title="The design of preemptible read-copy-update" +,month="October" +,day="8" +,year="2007" +,note="Available: +\url{http://lwn.net/Articles/253651/} +[Viewed October 25, 2007]" +,annotation=" + LWN article describing the design of preemptible RCU. +" +} + +######################################################################## +# +# "What is RCU?" LWN series. +# + +@unpublished{PaulEMcKenney2007WhatIsRCUFundamentally +,Author="Paul E. McKenney and Jonathan Walpole" +,Title="What is {RCU}, Fundamentally?" +,month="December" +,day="17" +,year="2007" +,note="Available: +\url{http://lwn.net/Articles/262464/} +[Viewed December 27, 2007]" +,annotation=" + Lays out the three basic components of RCU: (1) publish-subscribe, + (2) wait for pre-existing readers to complete, and (2) maintain + multiple versions. +" +} + +@unpublished{PaulEMcKenney2008WhatIsRCUUsage +,Author="Paul E. McKenney" +,Title="What is {RCU}? Part 2: Usage" +,month="January" +,day="4" +,year="2008" +,note="Available: +\url{http://lwn.net/Articles/263130/} +[Viewed January 4, 2008]" +,annotation=" + Lays out six uses of RCU: + 1. RCU is a Reader-Writer Lock Replacement + 2. RCU is a Restricted Reference-Counting Mechanism + 3. RCU is a Bulk Reference-Counting Mechanism + 4. RCU is a Poor Man's Garbage Collector + 5. RCU is a Way of Providing Existence Guarantees + 6. RCU is a Way of Waiting for Things to Finish +" +} + +@unpublished{PaulEMcKenney2008WhatIsRCUAPI +,Author="Paul E. McKenney" +,Title="{RCU} part 3: the {RCU} {API}" +,month="January" +,day="17" +,year="2008" +,note="Available: +\url{http://lwn.net/Articles/264090/} +[Viewed January 10, 2008]" +,annotation=" + Gives an overview of the Linux-kernel RCU API and a brief annotated RCU + bibliography. +" +} + +@article{DinakarGuniguntala2008IBMSysJ +,author="D. Guniguntala and P. E. McKenney and J. Triplett and J. Walpole" +,title="The read-copy-update mechanism for supporting real-time applications on shared-memory multiprocessor systems with {Linux}" +,Year="2008" +,Month="April" +,journal="IBM Systems Journal" +,volume="47" +,number="2" +,pages="@@-@@" +,annotation=" + RCU, realtime RCU, sleepable RCU, performance. +" +} diff --git a/Documentation/RCU/checklist.txt b/Documentation/RCU/checklist.txt index 42b01bc2e1b4..cf5562cbe356 100644 --- a/Documentation/RCU/checklist.txt +++ b/Documentation/RCU/checklist.txt @@ -13,10 +13,13 @@ over a rather long period of time, but improvements are always welcome! detailed performance measurements show that RCU is nonetheless the right tool for the job. - The other exception would be where performance is not an issue, - and RCU provides a simpler implementation. An example of this - situation is the dynamic NMI code in the Linux 2.6 kernel, - at least on architectures where NMIs are rare. + Another exception is where performance is not an issue, and RCU + provides a simpler implementation. An example of this situation + is the dynamic NMI code in the Linux 2.6 kernel, at least on + architectures where NMIs are rare. + + Yet another exception is where the low real-time latency of RCU's + read-side primitives is critically important. 1. Does the update code have proper mutual exclusion? @@ -39,9 +42,10 @@ over a rather long period of time, but improvements are always welcome! 2. Do the RCU read-side critical sections make proper use of rcu_read_lock() and friends? These primitives are needed - to suppress preemption (or bottom halves, in the case of - rcu_read_lock_bh()) in the read-side critical sections, - and are also an excellent aid to readability. + to prevent grace periods from ending prematurely, which + could result in data being unceremoniously freed out from + under your read-side code, which can greatly increase the + actuarial risk of your kernel. As a rough rule of thumb, any dereference of an RCU-protected pointer must be covered by rcu_read_lock() or rcu_read_lock_bh() @@ -54,15 +58,30 @@ over a rather long period of time, but improvements are always welcome! be running while updates are in progress. There are a number of ways to handle this concurrency, depending on the situation: - a. Make updates appear atomic to readers. For example, + a. Use the RCU variants of the list and hlist update + primitives to add, remove, and replace elements on an + RCU-protected list. Alternatively, use the RCU-protected + trees that have been added to the Linux kernel. + + This is almost always the best approach. + + b. Proceed as in (a) above, but also maintain per-element + locks (that are acquired by both readers and writers) + that guard per-element state. Of course, fields that + the readers refrain from accessing can be guarded by the + update-side lock. + + This works quite well, also. + + c. Make updates appear atomic to readers. For example, pointer updates to properly aligned fields will appear atomic, as will individual atomic primitives. Operations performed under a lock and sequences of multiple atomic primitives will -not- appear to be atomic. - This is almost always the best approach. + This can work, but is starting to get a bit tricky. - b. Carefully order the updates and the reads so that + d. Carefully order the updates and the reads so that readers see valid data at all phases of the update. This is often more difficult than it sounds, especially given modern CPUs' tendency to reorder memory references. @@ -123,18 +142,22 @@ over a rather long period of time, but improvements are always welcome! when publicizing a pointer to a structure that can be traversed by an RCU read-side critical section. -5. If call_rcu(), or a related primitive such as call_rcu_bh(), - is used, the callback function must be written to be called - from softirq context. In particular, it cannot block. +5. If call_rcu(), or a related primitive such as call_rcu_bh() or + call_rcu_sched(), is used, the callback function must be + written to be called from softirq context. In particular, + it cannot block. 6. Since synchronize_rcu() can block, it cannot be called from - any sort of irq context. + any sort of irq context. Ditto for synchronize_sched() and + synchronize_srcu(). 7. If the updater uses call_rcu(), then the corresponding readers must use rcu_read_lock() and rcu_read_unlock(). If the updater uses call_rcu_bh(), then the corresponding readers must use - rcu_read_lock_bh() and rcu_read_unlock_bh(). Mixing things up - will result in confusion and broken kernels. + rcu_read_lock_bh() and rcu_read_unlock_bh(). If the updater + uses call_rcu_sched(), then the corresponding readers must + disable preemption. Mixing things up will result in confusion + and broken kernels. One exception to this rule: rcu_read_lock() and rcu_read_unlock() may be substituted for rcu_read_lock_bh() and rcu_read_unlock_bh() @@ -143,9 +166,9 @@ over a rather long period of time, but improvements are always welcome! such cases is a must, of course! And the jury is still out on whether the increased speed is worth it. -8. Although synchronize_rcu() is a bit slower than is call_rcu(), - it usually results in simpler code. So, unless update - performance is critically important or the updaters cannot block, +8. Although synchronize_rcu() is slower than is call_rcu(), it + usually results in simpler code. So, unless update performance + is critically important or the updaters cannot block, synchronize_rcu() should be used in preference to call_rcu(). An especially important property of the synchronize_rcu() @@ -187,23 +210,23 @@ over a rather long period of time, but improvements are always welcome! number of updates per grace period. 9. All RCU list-traversal primitives, which include - list_for_each_rcu(), list_for_each_entry_rcu(), + rcu_dereference(), list_for_each_rcu(), list_for_each_entry_rcu(), list_for_each_continue_rcu(), and list_for_each_safe_rcu(), - must be within an RCU read-side critical section. RCU + must be either within an RCU read-side critical section or + must be protected by appropriate update-side locks. RCU read-side critical sections are delimited by rcu_read_lock() and rcu_read_unlock(), or by similar primitives such as rcu_read_lock_bh() and rcu_read_unlock_bh(). - Use of the _rcu() list-traversal primitives outside of an - RCU read-side critical section causes no harm other than - a slight performance degradation on Alpha CPUs. It can - also be quite helpful in reducing code bloat when common - code is shared between readers and updaters. + The reason that it is permissible to use RCU list-traversal + primitives when the update-side lock is held is that doing so + can be quite helpful in reducing code bloat when common code is + shared between readers and updaters. 10. Conversely, if you are in an RCU read-side critical section, - you -must- use the "_rcu()" variants of the list macros. - Failing to do so will break Alpha and confuse people reading - your code. + and you don't hold the appropriate update-side lock, you -must- + use the "_rcu()" variants of the list macros. Failing to do so + will break Alpha and confuse people reading your code. 11. Note that synchronize_rcu() -only- guarantees to wait until all currently executing rcu_read_lock()-protected RCU read-side @@ -230,6 +253,14 @@ over a rather long period of time, but improvements are always welcome! must use whatever locking or other synchronization is required to safely access and/or modify that data structure. + RCU callbacks are -usually- executed on the same CPU that executed + the corresponding call_rcu(), call_rcu_bh(), or call_rcu_sched(), + but are by -no- means guaranteed to be. For example, if a given + CPU goes offline while having an RCU callback pending, then that + RCU callback will execute on some surviving CPU. (If this was + not the case, a self-spawning RCU callback would prevent the + victim CPU from ever going offline.) + 14. SRCU (srcu_read_lock(), srcu_read_unlock(), and synchronize_srcu()) may only be invoked from process context. Unlike other forms of RCU, it -is- permissible to block in an SRCU read-side critical diff --git a/Documentation/RCU/whatisRCU.txt b/Documentation/RCU/whatisRCU.txt index e0d6d99b8f9b..e04d643a9f57 100644 --- a/Documentation/RCU/whatisRCU.txt +++ b/Documentation/RCU/whatisRCU.txt @@ -1,3 +1,11 @@ +Please note that the "What is RCU?" LWN series is an excellent place +to start learning about RCU: + +1. What is RCU, Fundamentally? http://lwn.net/Articles/262464/ +2. What is RCU? Part 2: Usage http://lwn.net/Articles/263130/ +3. RCU part 3: the RCU API http://lwn.net/Articles/264090/ + + What is RCU? RCU is a synchronization mechanism that was added to the Linux kernel @@ -772,26 +780,18 @@ Linux-kernel source code, but it helps to have a full list of the APIs, since there does not appear to be a way to categorize them in docbook. Here is the list, by category. -Markers for RCU read-side critical sections: - - rcu_read_lock - rcu_read_unlock - rcu_read_lock_bh - rcu_read_unlock_bh - srcu_read_lock - srcu_read_unlock - RCU pointer/list traversal: rcu_dereference + list_for_each_entry_rcu + hlist_for_each_entry_rcu + list_for_each_rcu (to be deprecated in favor of list_for_each_entry_rcu) - list_for_each_entry_rcu list_for_each_continue_rcu (to be deprecated in favor of new list_for_each_entry_continue_rcu) - hlist_for_each_entry_rcu -RCU pointer update: +RCU pointer/list update: rcu_assign_pointer list_add_rcu @@ -799,16 +799,36 @@ RCU pointer update: list_del_rcu list_replace_rcu hlist_del_rcu + hlist_add_after_rcu + hlist_add_before_rcu hlist_add_head_rcu + hlist_replace_rcu + list_splice_init_rcu() -RCU grace period: +RCU: Critical sections Grace period Barrier + + rcu_read_lock synchronize_net rcu_barrier + rcu_read_unlock synchronize_rcu + call_rcu + + +bh: Critical sections Grace period Barrier + + rcu_read_lock_bh call_rcu_bh rcu_barrier_bh + rcu_read_unlock_bh + + +sched: Critical sections Grace period Barrier + + [preempt_disable] synchronize_sched rcu_barrier_sched + [and friends] call_rcu_sched + + +SRCU: Critical sections Grace period Barrier + + srcu_read_lock synchronize_srcu N/A + srcu_read_unlock - synchronize_net - synchronize_sched - synchronize_rcu - synchronize_srcu - call_rcu - call_rcu_bh See the comment headers in the source code (or the docbook generated from them) for more information. diff --git a/Documentation/immediate.txt b/Documentation/immediate.txt new file mode 100644 index 000000000000..281e07341c99 --- /dev/null +++ b/Documentation/immediate.txt @@ -0,0 +1,221 @@ + Using the Immediate Values + + Mathieu Desnoyers + + +This document introduces Immediate Values and their use. + + +* Purpose of immediate values + +An immediate value is used to compile into the kernel variables that sit within +the instruction stream. They are meant to be rarely updated but read often. +Using immediate values for these variables will save cache lines. + +This infrastructure is specialized in supporting dynamic patching of the values +in the instruction stream when multiple CPUs are running without disturbing the +normal system behavior. + +Compiling code meant to be rarely enabled at runtime can be done using +if (unlikely(imv_read(var))) as condition surrounding the code. The +smallest data type required for the test (an 8 bits char) is preferred, since +some architectures, such as powerpc, only allow up to 16 bits immediate values. + + +* Usage + +In order to use the "immediate" macros, you should include linux/immediate.h. + +#include <linux/immediate.h> + +DEFINE_IMV(char, this_immediate); +EXPORT_IMV_SYMBOL(this_immediate); + + +And use, in the body of a function: + +Use imv_set(this_immediate) to set the immediate value. + +Use imv_read(this_immediate) to read the immediate value. + +The immediate mechanism supports inserting multiple instances of the same +immediate. Immediate values can be put in inline functions, inlined static +functions, and unrolled loops. + +If you have to read the immediate values from a function declared as __exit, you +should explicitly use _imv_read(), which will fall back on a global variable +read. Failing to do so will leave a reference to the __exit section in kernel +without module unload support. imv_read() in the __init section is supported. + +You can choose to set an initial static value to the immediate by using, for +instance: + +DEFINE_IMV(long, myptr) = 10; + + +* Optimization for a given architecture + +One can implement optimized immediate values for a given architecture by +replacing asm-$ARCH/immediate.h. + + +* Performance improvement + + + * Memory hit for a data-based branch + +Here are the results on a 3GHz Pentium 4: + +number of tests: 100 +number of branches per test: 100000 +memory hit cycles per iteration (mean): 636.611 +L1 cache hit cycles per iteration (mean): 89.6413 +instruction stream based test, cycles per iteration (mean): 85.3438 +Just getting the pointer from a modulo on a pseudo-random value, doing + nothing with it, cycles per iteration (mean): 77.5044 + +So: +Base case: 77.50 cycles +instruction stream based test: +7.8394 cycles +L1 cache hit based test: +12.1369 cycles +Memory load based test: +559.1066 cycles + +So let's say we have a ping flood coming at +(14014 packets transmitted, 14014 received, 0% packet loss, time 1826ms) +7674 packets per second. If we put 2 markers for irq entry/exit, it +brings us to 15348 markers sites executed per second. + +(15348 exec/s) * (559 cycles/exec) / (3G cycles/s) = 0.0029 +We therefore have a 0.29% slowdown just on this case. + +Compared to this, the instruction stream based test will cause a +slowdown of: + +(15348 exec/s) * (7.84 cycles/exec) / (3G cycles/s) = 0.00004 +For a 0.004% slowdown. + +If we plan to use this for memory allocation, spinlock, and all sorts of +very high event rate tracing, we can assume it will execute 10 to 100 +times more sites per second, which brings us to 0.4% slowdown with the +instruction stream based test compared to 29% slowdown with the memory +load based test on a system with high memory pressure. + + + + * Markers impact under heavy memory load + +Running a kernel with my LTTng instrumentation set, in a test that +generates memory pressure (from userspace) by trashing L1 and L2 caches +between calls to getppid() (note: syscall_trace is active and calls +a marker upon syscall entry and syscall exit; markers are disarmed). +This test is done in user-space, so there are some delays due to IRQs +coming and to the scheduler. (UP 2.6.22-rc6-mm1 kernel, task with -20 +nice level) + +My first set of results: Linear cache trashing, turned out not to be +very interesting, because it seems like the linearity of the memset on a +full array is somehow detected and it does not "really" trash the +caches. + +Now the most interesting result: Random walk L1 and L2 trashing +surrounding a getppid() call. + +- Markers compiled out (but syscall_trace execution forced) +number of tests: 10000 +No memory pressure +Reading timestamps takes 108.033 cycles +getppid: 1681.4 cycles +With memory pressure +Reading timestamps takes 102.938 cycles +getppid: 15691.6 cycles + + +- With the immediate values based markers: +number of tests: 10000 +No memory pressure +Reading timestamps takes 108.006 cycles +getppid: 1681.84 cycles +With memory pressure +Reading timestamps takes 100.291 cycles +getppid: 11793 cycles + + +- With global variables based markers: +number of tests: 10000 +No memory pressure +Reading timestamps takes 107.999 cycles +getppid: 1669.06 cycles +With memory pressure +Reading timestamps takes 102.839 cycles +getppid: 12535 cycles + +The result is quite interesting in that the kernel is slower without +markers than with markers. I explain it by the fact that the data +accessed is not laid out in the same manner in the cache lines when the +markers are compiled in or out. It seems that it aligns the function's +data better to compile-in the markers in this case. + +But since the interesting comparison is between the immediate values and +global variables based markers, and because they share the same memory +layout, except for the movl being replaced by a movz, we see that the +global variable based markers (2 markers) adds 742 cycles to each system +call (syscall entry and exit are traced and memory locations for both +global variables lie on the same cache line). + + +- Test redone with less iterations, but with error estimates + +10 runs of 100 iterations each: Tests done on a 3GHz P4. Here I run getppid with +syscall trace inactive, comparing the case with memory pressure and without +memory pressure. (sorry, my system is not setup to execute syscall_trace this +time, but it will make the point anyway). + +No memory pressure +Reading timestamps: 150.92 cycles, std dev. 1.01 cycles +getppid: 1462.09 cycles, std dev. 18.87 cycles + +With memory pressure +Reading timestamps: 578.22 cycles, std dev. 269.51 cycles +getppid: 17113.33 cycles, std dev. 1655.92 cycles + + +Now for memory read timing: (10 runs, branches per test: 100000) +Memory read based branch: + 644.09 cycles, std dev. 11.39 cycles +L1 cache hit based branch: + 88.16 cycles, std dev. 1.35 cycles + + +So, now that we have the raw results, let's calculate: + +Memory read: +644.09 +/- 11.39 - 88.16 +/- 1.35 = 555.93 +/- 11.46 cycles + +Getppid without memory pressure: +1462.09 +/- 18.87 - 150.92 +/- 1.01 = 1311.17 +/- 18.90 cycles + +Getppid with memory pressure: +17113.33 +/- 1655.92 - 578.22 +/- 269.51 = 16535.11 +/- 1677.71 cycles + +Therefore, if we add 2 markers not based on immediate values to the getppid +code, which would add 2 memory reads, we would add +2 * 555.93 +/- 12.74 = 1111.86 +/- 25.48 cycles + +Therefore, + +1111.86 +/- 25.48 / 16535.11 +/- 1677.71 = 0.0672 + relative error: sqrt(((25.48/1111.86)^2)+((1677.71/16535.11)^2)) + = 0.1040 + absolute error: 0.1040 * 0.0672 = 0.0070 + +Therefore: 0.0672 +/- 0.0070 * 100% = 6.72 +/- 0.70 % + +We can therefore affirm that adding 2 markers to getppid, on a system with high +memory pressure, would have a performance hit of at least 6.0% on the system +call time, all within the uncertainty limits of these tests. The same applies to +other kernel code paths. The smaller those code paths are, the highest the +impact ratio will be. + +Therefore, not only is it interesting to use the immediate values to dynamically +activate dormant code such as the markers, but I think it should also be +considered as a replacement for many of the "read-mostly" static variables. diff --git a/Documentation/kernel-parameters.txt b/Documentation/kernel-parameters.txt index 388e961fb93c..41e5c635fa15 100644 --- a/Documentation/kernel-parameters.txt +++ b/Documentation/kernel-parameters.txt @@ -1988,6 +1988,9 @@ and is between 256 and 4096 characters. It is defined in the file snd-ymfpci= [HW,ALSA] + softlockup_panic= + [KNL] Should the soft-lockup detector generate panics. + sonypi.*= [HW] Sony Programmable I/O Control Device driver See Documentation/sonypi.txt diff --git a/Documentation/markers.txt b/Documentation/markers.txt index d9f50a19fa0c..e43427b8596d 100644 --- a/Documentation/markers.txt +++ b/Documentation/markers.txt @@ -15,10 +15,12 @@ provide at runtime. A marker can be "on" (a probe is connected to it) or "off" (no probe is attached). When a marker is "off" it has no effect, except for adding a tiny time penalty (checking a condition for a branch) and space penalty (adding a few bytes for the function call at the end of the -instrumented function and adds a data structure in a separate section). When a -marker is "on", the function you provide is called each time the marker is -executed, in the execution context of the caller. When the function provided -ends its execution, it returns to the caller (continuing from the marker site). +instrumented function and adds a data structure in a separate section). The +immediate values are used to minimize the impact on data cache, encoding the +condition in the instruction stream. When a marker is "on", the function you +provide is called each time the marker is executed, in the execution context of +the caller. When the function provided ends its execution, it returns to the +caller (continuing from the marker site). You can put markers at important locations in the code. Markers are lightweight hooks that can pass an arbitrary number of parameters, @@ -69,6 +71,13 @@ a printk warning which identifies the inconsistency: "Format mismatch for probe probe_name (format), marker (format)" +* Optimization for a given architecture + +To force use of a non-optimized version of the markers, _trace_mark() should be +used. It takes the same parameters as the normal markers, but it does not use +the immediate values based on code patching. + + * Probe / marker example See the example provided in samples/markers/src diff --git a/arch/alpha/kernel/irq.c b/arch/alpha/kernel/irq.c index facf82a5499a..741028c13b11 100644 --- a/arch/alpha/kernel/irq.c +++ b/arch/alpha/kernel/irq.c @@ -51,7 +51,7 @@ select_smp_affinity(unsigned int irq) if (!irq_desc[irq].chip->set_affinity || irq_user_affinity[irq]) return 1; - while (!cpu_possible(cpu)) + while (!cpu_possible(cpu) || !cpu_system(cpu)) cpu = (cpu < (NR_CPUS-1) ? cpu + 1 : 0); last_cpu = cpu; diff --git a/arch/ia64/kernel/kprobes.c b/arch/ia64/kernel/kprobes.c index 233434f4f88f..7bbcaa804ef5 100644 --- a/arch/ia64/kernel/kprobes.c +++ b/arch/ia64/kernel/kprobes.c @@ -672,9 +672,7 @@ void __kprobes arch_disarm_kprobe(struct kprobe *p) void __kprobes arch_remove_kprobe(struct kprobe *p) { - mutex_lock(&kprobe_mutex); free_insn_slot(p->ainsn.insn, p->ainsn.inst_flag & INST_FLAG_BOOSTABLE); - mutex_unlock(&kprobe_mutex); } /* * We are resuming execution after a single step fault, so the pt_regs diff --git a/arch/ia64/sn/kernel/irq.c b/arch/ia64/sn/kernel/irq.c index 53351c3cd7b1..96c31b4180c3 100644 --- a/arch/ia64/sn/kernel/irq.c +++ b/arch/ia64/sn/kernel/irq.c @@ -11,6 +11,7 @@ #include <linux/irq.h> #include <linux/spinlock.h> #include <linux/init.h> +#include <linux/rculist.h> #include <asm/sn/addrs.h> #include <asm/sn/arch.h> #include <asm/sn/intr.h> diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 3934e2659407..a6b28d7412b6 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -110,6 +110,7 @@ config PPC select HAVE_KPROBES select HAVE_KRETPROBES select HAVE_LMB + select HAVE_IMMEDIATE config EARLY_PRINTK bool diff --git a/arch/powerpc/kernel/Makefile b/arch/powerpc/kernel/Makefile index d14cebf62bb0..b53dff108237 100644 --- a/arch/powerpc/kernel/Makefile +++ b/arch/powerpc/kernel/Makefile @@ -45,6 +45,7 @@ obj-$(CONFIG_HIBERNATION) += swsusp.o suspend.o \ obj64-$(CONFIG_HIBERNATION) += swsusp_asm64.o obj-$(CONFIG_MODULES) += module_$(CONFIG_WORD_SIZE).o obj-$(CONFIG_44x) += cpu_setup_44x.o +obj-$(CONFIG_IMMEDIATE) += immediate.o ifeq ($(CONFIG_PPC_MERGE),y) diff --git a/arch/powerpc/kernel/immediate.c b/arch/powerpc/kernel/immediate.c new file mode 100644 index 000000000000..3413933b2166 --- /dev/null +++ b/arch/powerpc/kernel/immediate.c @@ -0,0 +1,70 @@ +/* + * Powerpc optimized immediate values enabling/disabling. + * + * Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca> + */ + +#include <linux/module.h> +#include <linux/immediate.h> +#include <linux/string.h> +#include <linux/kprobes.h> +#include <asm/cacheflush.h> +#include <asm/page.h> + +#define LI_OPCODE_LEN 2 + +/** + * arch_imv_update - update one immediate value + * @imv: pointer of type const struct __imv to update + * @early: early boot (1), normal (0) + * + * Update one immediate value. Must be called with imv_mutex held. + */ +int arch_imv_update(const struct __imv *imv, int early) +{ +#ifdef CONFIG_KPROBES + kprobe_opcode_t *insn; + /* + * Fail if a kprobe has been set on this instruction. + * (TODO: we could eventually do better and modify all the (possibly + * nested) kprobes for this site if kprobes had an API for this. + */ + switch (imv->size) { + case 1: /* The uint8_t points to the 3rd byte of the + * instruction */ + insn = (void *)(imv->imv - 1 - LI_OPCODE_LEN); + break; + case 2: insn = (void *)(imv->imv - LI_OPCODE_LEN); + break; + default: + return -EINVAL; + } + + if (unlikely(!early && *insn == BREAKPOINT_INSTRUCTION)) { + printk(KERN_WARNING "Immediate value in conflict with kprobe. " + "Variable at %p, " + "instruction at %p, size %lu\n", + (void *)imv->imv, + (void *)imv->var, imv->size); + return -EBUSY; + } +#endif + + /* + * If the variable and the instruction have the same value, there is + * nothing to do. + */ + switch (imv->size) { + case 1: if (*(uint8_t *)imv->imv == *(uint8_t *)imv->var) + return 0; + *(uint8_t *)imv->imv = *(uint8_t *)imv->var; + break; + case 2: if (*(uint16_t *)imv->imv == *(uint16_t *)imv->var) + return 0; + *(uint16_t *)imv->imv = *(uint16_t *)imv->var; + break; + default:return -EINVAL; + } + flush_icache_range(imv->imv, imv->imv + imv->size); + return 0; +} diff --git a/arch/powerpc/kernel/kprobes.c b/arch/powerpc/kernel/kprobes.c index c176c513566b..8ade996dd771 100644 --- a/arch/powerpc/kernel/kprobes.c +++ b/arch/powerpc/kernel/kprobes.c @@ -88,9 +88,7 @@ void __kprobes arch_disarm_kprobe(struct kprobe *p) void __kprobes arch_remove_kprobe(struct kprobe *p) { - mutex_lock(&kprobe_mutex); free_insn_slot(p->ainsn.insn, 0); - mutex_unlock(&kprobe_mutex); } static void __kprobes prepare_singlestep(struct kprobe *p, struct pt_regs *regs) diff --git a/arch/s390/kernel/kprobes.c b/arch/s390/kernel/kprobes.c index ed04d1372d5d..5a1c7e4a6f7a 100644 --- a/arch/s390/kernel/kprobes.c +++ b/arch/s390/kernel/kprobes.c @@ -220,9 +220,7 @@ void __kprobes arch_disarm_kprobe(struct kprobe *p) void __kprobes arch_remove_kprobe(struct kprobe *p) { - mutex_lock(&kprobe_mutex); free_insn_slot(p->ainsn.insn, 0); - mutex_unlock(&kprobe_mutex); } static void __kprobes prepare_singlestep(struct kprobe *p, struct pt_regs *regs) diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 32fe68e8f028..fde03752e300 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -26,6 +26,7 @@ config X86 select HAVE_FTRACE select HAVE_KVM if ((X86_32 && !X86_VOYAGER && !X86_VISWS && !X86_NUMAQ) || X86_64) select HAVE_ARCH_KGDB if !X86_VOYAGER + select HAVE_IMMEDIATE config DEFCONFIG_LIST string @@ -129,7 +130,7 @@ config ARCH_HAS_CACHE_LINE_SIZE def_bool y config HAVE_SETUP_PER_CPU_AREA - def_bool X86_64 || (X86_SMP && !X86_VOYAGER) + def_bool X86_64_SMP || (X86_SMP && !X86_VOYAGER) config HAVE_CPUMASK_OF_CPU_MAP def_bool X86_64_SMP @@ -548,20 +549,35 @@ config SWIOTLB config IOMMU_HELPER def_bool (CALGARY_IOMMU || GART_IOMMU || SWIOTLB) +config MAXSMP + bool "Configure Maximum number of SMP Processors and NUMA Nodes" + depends on X86_64 && SMP + default n + help + Configure maximum number of CPUS and NUMA Nodes for this architecture. + If unsure, say N. +if MAXSMP config NR_CPUS - int "Maximum number of CPUs (2-255)" - range 2 255 + int + default "4096" +endif + +if !MAXSMP +config NR_CPUS + int "Maximum number of CPUs (2-4096)" + range 2 4096 depends on SMP default "32" if X86_NUMAQ || X86_SUMMIT || X86_BIGSMP || X86_ES7000 default "8" help This allows you to specify the maximum number of CPUs which this - kernel will support. The maximum supported value is 255 and the + kernel will support. The maximum supported value is 4096 and the minimum value which makes sense is 2. This is purely to save memory - each supported CPU adds approximately eight kilobytes to the kernel image. +endif config SCHED_SMT bool "SMT (Hyperthreading) scheduler support" @@ -952,13 +968,25 @@ config NUMA_EMU into virtual nodes when booted with "numa=fake=N", where N is the number of nodes. This is only useful for debugging. +if MAXSMP + +config NODES_SHIFT + int + default "9" +endif + +if !MAXSMP config NODES_SHIFT - int "Max num nodes shift(1-15)" - range 1 15 if X86_64 + int "Maximum NUMA Nodes (as a power of 2)" + range 1 9 if X86_64 default "6" if X86_64 default "4" if X86_NUMAQ default "3" depends on NEED_MULTIPLE_NODES + help + Specify the maximum number of NUMA Nodes available on the target + system. Increases memory reserved to accomodate various tables. +endif config HAVE_ARCH_BOOTMEM_NODE def_bool y diff --git a/arch/x86/Kconfig.debug b/arch/x86/Kconfig.debug index e7aa050ba539..2fac95882250 100644 --- a/arch/x86/Kconfig.debug +++ b/arch/x86/Kconfig.debug @@ -56,7 +56,7 @@ config DEBUG_PAGEALLOC config DEBUG_PER_CPU_MAPS bool "Debug access to per_cpu maps" depends on DEBUG_KERNEL - depends on X86_64_SMP + depends on X86_SMP default n help Say Y to verify that the per_cpu map being accessed has @@ -241,9 +241,13 @@ config IO_DELAY_UDELAY config IO_DELAY_NONE bool "no port-IO delay" + depends on BROKEN && KERNEL_DEBUG help No port-IO delay. Will break on old boxes that require port-IO - delay for certain operations. Should work on most new machines. + delay for certain operations. Should work on most new machines + but not on all. There's no good way to determine whether this + will break anything so only enable it if you know what you are + doing. endchoice diff --git a/arch/x86/kernel/Makefile b/arch/x86/kernel/Makefile index 172557877ada..5f46d56741e2 100644 --- a/arch/x86/kernel/Makefile +++ b/arch/x86/kernel/Makefile @@ -6,6 +6,14 @@ extra-y := head_$(BITS).o head$(BITS).o init_task.o vmlinux.lds CPPFLAGS_vmlinux.lds += -U$(UTS_MACHINE) +ifdef CONFIG_FTRACE +# Do not profile debug utilities +ORIG_CFLAGS := $(KBUILD_CFLAGS) +KBUILD_CFLAGS = $(if $(filter-out tsc_64 tsc_32 rtc,$(basename $(notdir $@))), \ + $(ORIG_CFLAGS), \ + $(subst -pg,,$(ORIG_CFLAGS))) +endif + # # vsyscalls (which work on the user stack) should have # no stack-protector checks: @@ -67,6 +75,7 @@ obj-y += vsmp_64.o obj-$(CONFIG_KPROBES) += kprobes.o obj-$(CONFIG_MODULES) += module_$(BITS).o obj-$(CONFIG_ACPI_SRAT) += srat_32.o +obj-$(CONFIG_IMMEDIATE) += immediate.o obj-$(CONFIG_EFI) += efi.o efi_$(BITS).o efi_stub_$(BITS).o obj-$(CONFIG_DOUBLEFAULT) += doublefault_32.o obj-$(CONFIG_KGDB) += kgdb.o diff --git a/arch/x86/kernel/apic_32.c b/arch/x86/kernel/apic_32.c index d5767cb19d56..7bf5649f3106 100644 --- a/arch/x86/kernel/apic_32.c +++ b/arch/x86/kernel/apic_32.c @@ -52,9 +52,6 @@ unsigned long mp_lapic_addr; -DEFINE_PER_CPU(u16, x86_bios_cpu_apicid) = BAD_APICID; -EXPORT_PER_CPU_SYMBOL(x86_bios_cpu_apicid); - /* * Knob to control our willingness to enable the local APIC. * @@ -1538,9 +1535,9 @@ void __cpuinit generic_processor_info(int apicid, int version) } #ifdef CONFIG_SMP /* are we being called early in kernel startup? */ - if (x86_cpu_to_apicid_early_ptr) { - u16 *cpu_to_apicid = x86_cpu_to_apicid_early_ptr; - u16 *bios_cpu_apicid = x86_bios_cpu_apicid_early_ptr; + if (early_per_cpu_ptr(x86_cpu_to_apicid)) { + u16 *cpu_to_apicid = early_per_cpu_ptr(x86_cpu_to_apicid); + u16 *bios_cpu_apicid = early_per_cpu_ptr(x86_bios_cpu_apicid); cpu_to_apicid[cpu] = apicid; bios_cpu_apicid[cpu] = apicid; diff --git a/arch/x86/kernel/apic_64.c b/arch/x86/kernel/apic_64.c index 3b3584b3235d..30f3f6ae1d85 100644 --- a/arch/x86/kernel/apic_64.c +++ b/arch/x86/kernel/apic_64.c @@ -87,9 +87,6 @@ static unsigned long apic_phys; unsigned long mp_lapic_addr; -DEFINE_PER_CPU(u16, x86_bios_cpu_apicid) = BAD_APICID; -EXPORT_PER_CPU_SYMBOL(x86_bios_cpu_apicid); - unsigned int __cpuinitdata maxcpus = NR_CPUS; /* * Get the LAPIC version @@ -1091,9 +1088,9 @@ void __cpuinit generic_processor_info(int apicid, int version) cpu = 0; } /* are we being called early in kernel startup? */ - if (x86_cpu_to_apicid_early_ptr) { - u16 *cpu_to_apicid = x86_cpu_to_apicid_early_ptr; - u16 *bios_cpu_apicid = x86_bios_cpu_apicid_early_ptr; + if (early_per_cpu_ptr(x86_cpu_to_apicid)) { + u16 *cpu_to_apicid = early_per_cpu_ptr(x86_cpu_to_apicid); + u16 *bios_cpu_apicid = early_per_cpu_ptr(x86_bios_cpu_apicid); cpu_to_apicid[cpu] = apicid; bios_cpu_apicid[cpu] = apicid; @@ -1269,7 +1266,7 @@ __cpuinit int apic_is_clustered_box(void) if ((boot_cpu_data.x86_vendor == X86_VENDOR_AMD) && !is_vsmp_box()) return 0; - bios_cpu_apicid = x86_bios_cpu_apicid_early_ptr; + bios_cpu_apicid = early_per_cpu_ptr(x86_bios_cpu_apicid); bitmap_zero(clustermap, NUM_APIC_CLUSTERS); for (i = 0; i < NR_CPUS; i++) { diff --git a/arch/x86/kernel/asm-offsets_32.c b/arch/x86/kernel/asm-offsets_32.c index 92588083950f..73474e00ad00 100644 --- a/arch/x86/kernel/asm-offsets_32.c +++ b/arch/x86/kernel/asm-offsets_32.c @@ -111,6 +111,7 @@ void foo(void) OFFSET(PV_IRQ_irq_disable, pv_irq_ops, irq_disable); OFFSET(PV_IRQ_irq_enable, pv_irq_ops, irq_enable); OFFSET(PV_CPU_iret, pv_cpu_ops, iret); + OFFSET(PV_CPU_nmi_return, pv_cpu_ops, nmi_return); OFFSET(PV_CPU_irq_enable_syscall_ret, pv_cpu_ops, irq_enable_syscall_ret); OFFSET(PV_CPU_read_cr0, pv_cpu_ops, read_cr0); #endif diff --git a/arch/x86/kernel/asm-offsets_64.c b/arch/x86/kernel/asm-offsets_64.c index f126c05d6170..a5bbec3ac53e 100644 --- a/arch/x86/kernel/asm-offsets_64.c +++ b/arch/x86/kernel/asm-offsets_64.c @@ -62,6 +62,7 @@ int main(void) OFFSET(PV_IRQ_irq_disable, pv_irq_ops, irq_disable); OFFSET(PV_IRQ_irq_enable, pv_irq_ops, irq_enable); OFFSET(PV_CPU_iret, pv_cpu_ops, iret); + OFFSET(PV_CPU_nmi_return, pv_cpu_ops, nmi_return); OFFSET(PV_CPU_irq_enable_syscall_ret, pv_cpu_ops, irq_enable_syscall_ret); OFFSET(PV_CPU_swapgs, pv_cpu_ops, swapgs); OFFSET(PV_MMU_read_cr2, pv_mmu_ops, read_cr2); diff --git a/arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c b/arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c index b0c8208df9fa..dd097b835839 100644 --- a/arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c +++ b/arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c @@ -202,7 +202,7 @@ static void drv_write(struct drv_cmd *cmd) cpumask_t saved_mask = current->cpus_allowed; unsigned int i; - for_each_cpu_mask(i, cmd->mask) { + for_each_cpu_mask_nr(i, cmd->mask) { set_cpus_allowed_ptr(current, &cpumask_of_cpu(i)); do_drv_write(cmd); } @@ -451,7 +451,7 @@ static int acpi_cpufreq_target(struct cpufreq_policy *policy, freqs.old = perf->states[perf->state].core_frequency * 1000; freqs.new = data->freq_table[next_state].frequency; - for_each_cpu_mask(i, cmd.mask) { + for_each_cpu_mask_nr(i, cmd.mask) { freqs.cpu = i; cpufreq_notify_transition(&freqs, CPUFREQ_PRECHANGE); } @@ -466,7 +466,7 @@ static int acpi_cpufreq_target(struct cpufreq_policy *policy, } } - for_each_cpu_mask(i, cmd.mask) { + for_each_cpu_mask_nr(i, cmd.mask) { freqs.cpu = i; cpufreq_notify_transition(&freqs, CPUFREQ_POSTCHANGE); } diff --git a/arch/x86/kernel/cpu/cpufreq/p4-clockmod.c b/arch/x86/kernel/cpu/cpufreq/p4-clockmod.c index 199e4e05e5dc..f1685fb91fbd 100644 --- a/arch/x86/kernel/cpu/cpufreq/p4-clockmod.c +++ b/arch/x86/kernel/cpu/cpufreq/p4-clockmod.c @@ -122,7 +122,7 @@ static int cpufreq_p4_target(struct cpufreq_policy *policy, return 0; /* notifiers */ - for_each_cpu_mask(i, policy->cpus) { + for_each_cpu_mask_nr(i, policy->cpus) { freqs.cpu = i; cpufreq_notify_transition(&freqs, CPUFREQ_PRECHANGE); } @@ -130,11 +130,11 @@ static int cpufreq_p4_target(struct cpufreq_policy *policy, /* run on each logical CPU, see section 13.15.3 of IA32 Intel Architecture Software * Developer's Manual, Volume 3 */ - for_each_cpu_mask(i, policy->cpus) + for_each_cpu_mask_nr(i, policy->cpus) cpufreq_p4_setdc(i, p4clockmod_table[newstate].index); /* notifiers */ - for_each_cpu_mask(i, policy->cpus) { + for_each_cpu_mask_nr(i, policy->cpus) { freqs.cpu = i; cpufreq_notify_transition(&freqs, CPUFREQ_POSTCHANGE); } diff --git a/arch/x86/kernel/cpu/cpufreq/powernow-k8.c b/arch/x86/kernel/cpu/cpufreq/powernow-k8.c index 46d4034d9f37..06d6eea5e07a 100644 --- a/arch/x86/kernel/cpu/cpufreq/powernow-k8.c +++ b/arch/x86/kernel/cpu/cpufreq/powernow-k8.c @@ -966,7 +966,7 @@ static int transition_frequency_fidvid(struct powernow_k8_data *data, unsigned i freqs.old = find_khz_freq_from_fid(data->currfid); freqs.new = find_khz_freq_from_fid(fid); - for_each_cpu_mask(i, *(data->available_cores)) { + for_each_cpu_mask_nr(i, *(data->available_cores)) { freqs.cpu = i; cpufreq_notify_transition(&freqs, CPUFREQ_PRECHANGE); } @@ -974,7 +974,7 @@ static int transition_frequency_fidvid(struct powernow_k8_data *data, unsigned i res = transition_fid_vid(data, fid, vid); freqs.new = find_khz_freq_from_fid(data->currfid); - for_each_cpu_mask(i, *(data->available_cores)) { + for_each_cpu_mask_nr(i, *(data->available_cores)) { freqs.cpu = i; cpufreq_notify_transition(&freqs, CPUFREQ_POSTCHANGE); } @@ -997,7 +997,7 @@ static int transition_frequency_pstate(struct powernow_k8_data *data, unsigned i freqs.old = find_khz_freq_from_pstate(data->powernow_table, data->currpstate); freqs.new = find_khz_freq_from_pstate(data->powernow_table, pstate); - for_each_cpu_mask(i, *(data->available_cores)) { + for_each_cpu_mask_nr(i, *(data->available_cores)) { freqs.cpu = i; cpufreq_notify_transition(&freqs, CPUFREQ_PRECHANGE); } @@ -1005,7 +1005,7 @@ static int transition_frequency_pstate(struct powernow_k8_data *data, unsigned i res = transition_pstate(data, pstate); freqs.new = find_khz_freq_from_pstate(data->powernow_table, pstate); - for_each_cpu_mask(i, *(data->available_cores)) { + for_each_cpu_mask_nr(i, *(data->available_cores)) { freqs.cpu = i; cpufreq_notify_transition(&freqs, CPUFREQ_POSTCHANGE); } diff --git a/arch/x86/kernel/cpu/cpufreq/speedstep-centrino.c b/arch/x86/kernel/cpu/cpufreq/speedstep-centrino.c index 908dd347c67e..8b0dd6f2a1ac 100644 --- a/arch/x86/kernel/cpu/cpufreq/speedstep-centrino.c +++ b/arch/x86/kernel/cpu/cpufreq/speedstep-centrino.c @@ -476,7 +476,7 @@ static int centrino_target (struct cpufreq_policy *policy, saved_mask = current->cpus_allowed; first_cpu = 1; cpus_clear(covered_cpus); - for_each_cpu_mask(j, online_policy_cpus) { + for_each_cpu_mask_nr(j, online_policy_cpus) { /* * Support for SMP systems. * Make sure we are running on CPU that wants to change freq @@ -517,7 +517,7 @@ static int centrino_target (struct cpufreq_policy *policy, dprintk("target=%dkHz old=%d new=%d msr=%04x\n", target_freq, freqs.old, freqs.new, msr); - for_each_cpu_mask(k, online_policy_cpus) { + for_each_cpu_mask_nr(k, online_policy_cpus) { freqs.cpu = k; cpufreq_notify_transition(&freqs, CPUFREQ_PRECHANGE); @@ -540,7 +540,7 @@ static int centrino_target (struct cpufreq_policy *policy, preempt_enable(); } - for_each_cpu_mask(k, online_policy_cpus) { + for_each_cpu_mask_nr(k, online_policy_cpus) { freqs.cpu = k; cpufreq_notify_transition(&freqs, CPUFREQ_POSTCHANGE); } @@ -554,7 +554,7 @@ static int centrino_target (struct cpufreq_policy *policy, */ if (!cpus_empty(covered_cpus)) { - for_each_cpu_mask(j, covered_cpus) { + for_each_cpu_mask_nr(j, covered_cpus) { set_cpus_allowed_ptr(current, &cpumask_of_cpu(j)); wrmsr(MSR_IA32_PERF_CTL, oldmsr, h); @@ -564,7 +564,7 @@ static int centrino_target (struct cpufreq_policy *policy, tmp = freqs.new; freqs.new = freqs.old; freqs.old = tmp; - for_each_cpu_mask(j, online_policy_cpus) { + for_each_cpu_mask_nr(j, online_policy_cpus) { freqs.cpu = j; cpufreq_notify_transition(&freqs, CPUFREQ_PRECHANGE); cpufreq_notify_transition(&freqs, CPUFREQ_POSTCHANGE); diff --git a/arch/x86/kernel/cpu/cpufreq/speedstep-ich.c b/arch/x86/kernel/cpu/cpufreq/speedstep-ich.c index 1b50244b1fdf..191f7263c61d 100644 --- a/arch/x86/kernel/cpu/cpufreq/speedstep-ich.c +++ b/arch/x86/kernel/cpu/cpufreq/speedstep-ich.c @@ -279,7 +279,7 @@ static int speedstep_target (struct cpufreq_policy *policy, cpus_allowed = current->cpus_allowed; - for_each_cpu_mask(i, policy->cpus) { + for_each_cpu_mask_nr(i, policy->cpus) { freqs.cpu = i; cpufreq_notify_transition(&freqs, CPUFREQ_PRECHANGE); } @@ -292,7 +292,7 @@ static int speedstep_target (struct cpufreq_policy *policy, /* allow to be run on all CPUs */ set_cpus_allowed_ptr(current, &cpus_allowed); - for_each_cpu_mask(i, policy->cpus) { + for_each_cpu_mask_nr(i, policy->cpus) { freqs.cpu = i; cpufreq_notify_transition(&freqs, CPUFREQ_POSTCHANGE); } diff --git a/arch/x86/kernel/cpu/intel_cacheinfo.c b/arch/x86/kernel/cpu/intel_cacheinfo.c index 2c8afafa18e8..a7b0f8f1736b 100644 --- a/arch/x86/kernel/cpu/intel_cacheinfo.c +++ b/arch/x86/kernel/cpu/intel_cacheinfo.c @@ -489,7 +489,7 @@ static void __cpuinit cache_remove_shared_cpu_map(unsigned int cpu, int index) int sibling; this_leaf = CPUID4_INFO_IDX(cpu, index); - for_each_cpu_mask(sibling, this_leaf->shared_cpu_map) { + for_each_cpu_mask_nr(sibling, this_leaf->shared_cpu_map) { sibling_leaf = CPUID4_INFO_IDX(sibling, index); cpu_clear(cpu, sibling_leaf->shared_cpu_map); } diff --git a/arch/x86/kernel/cpu/mcheck/mce_amd_64.c b/arch/x86/kernel/cpu/mcheck/mce_amd_64.c index 7c9a813e1193..88736cadbaa6 100644 --- a/arch/x86/kernel/cpu/mcheck/mce_amd_64.c +++ b/arch/x86/kernel/cpu/mcheck/mce_amd_64.c @@ -527,7 +527,7 @@ static __cpuinit int threshold_create_bank(unsigned int cpu, unsigned int bank) if (err) goto out_free; - for_each_cpu_mask(i, b->cpus) { + for_each_cpu_mask_nr(i, b->cpus) { if (i == cpu) continue; @@ -617,7 +617,7 @@ static void threshold_remove_bank(unsigned int cpu, int bank) #endif /* remove all sibling symlinks before unregistering */ - for_each_cpu_mask(i, b->cpus) { + for_each_cpu_mask_nr(i, b->cpus) { if (i == cpu) continue; diff --git a/arch/x86/kernel/entry_32.S b/arch/x86/kernel/entry_32.S index 75d03e497da3..4c7d571bebce 100644 --- a/arch/x86/kernel/entry_32.S +++ b/arch/x86/kernel/entry_32.S @@ -68,6 +68,8 @@ #define nr_syscalls ((syscall_table_size)/4) +#define HARDNMI_MASK 0x40000000 + #ifdef CONFIG_PREEMPT #define preempt_stop(clobbers) DISABLE_INTERRUPTS(clobbers); TRACE_IRQS_OFF #else @@ -232,8 +234,32 @@ END(ret_from_fork) # userspace resumption stub bypassing syscall exit tracing ALIGN RING0_PTREGS_FRAME + ret_from_exception: preempt_stop(CLBR_ANY) + GET_THREAD_INFO(%ebp) + movl PT_EFLAGS(%esp), %eax # mix EFLAGS and CS + movb PT_CS(%esp), %al + andl $(X86_EFLAGS_VM | SEGMENT_RPL_MASK), %eax + cmpl $USER_RPL, %eax + jae resume_userspace # returning to v8086 or userspace + testl $HARDNMI_MASK,TI_preempt_count(%ebp) + jz resume_kernel /* Not nested over NMI ? */ + testw $X86_EFLAGS_TF, PT_EFLAGS(%esp) + jnz resume_kernel /* + * If single-stepping an NMI handler, + * use the normal iret path instead of + * the popf/lret because lret would be + * single-stepped. It should not + * happen : it will reactivate NMIs + * prematurely. + */ + TRACE_IRQS_IRET + RESTORE_REGS + addl $4, %esp # skip orig_eax/error_code + CFI_ADJUST_CFA_OFFSET -4 + INTERRUPT_RETURN_NMI_SAFE + ret_from_intr: GET_THREAD_INFO(%ebp) check_userspace: @@ -873,6 +899,10 @@ ENTRY(native_iret) .previous END(native_iret) +ENTRY(native_nmi_return) + NATIVE_INTERRUPT_RETURN_NMI_SAFE # Should we deal with popf exception ? +END(native_nmi_return) + ENTRY(native_irq_enable_syscall_ret) sti sysexit diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S index 8f6b2e694e2a..17aeb96ec226 100644 --- a/arch/x86/kernel/entry_64.S +++ b/arch/x86/kernel/entry_64.S @@ -156,6 +156,8 @@ END(mcount) #endif /* CONFIG_DYNAMIC_FTRACE */ #endif /* CONFIG_FTRACE */ +#define HARDNMI_MASK 0x40000000 + #ifndef CONFIG_PREEMPT #define retint_kernel retint_restore_args #endif @@ -698,6 +700,9 @@ ENTRY(native_iret) .section __ex_table,"a" .quad native_iret, bad_iret .previous + +ENTRY(native_nmi_return) + NATIVE_INTERRUPT_RETURN_NMI_SAFE #endif .section .fixup,"ax" @@ -753,6 +758,23 @@ retint_signal: GET_THREAD_INFO(%rcx) jmp retint_check + /* Returning to kernel space from exception. */ + /* rcx: threadinfo. interrupts off. */ +ENTRY(retexc_kernel) + testl $HARDNMI_MASK,threadinfo_preempt_count(%rcx) + jz retint_kernel /* Not nested over NMI ? */ + testw $X86_EFLAGS_TF,EFLAGS-ARGOFFSET(%rsp) /* trap flag? */ + jnz retint_kernel /* + * If single-stepping an NMI handler, + * use the normal iret path instead of + * the popf/lret because lret would be + * single-stepped. It should not + * happen : it will reactivate NMIs + * prematurely. + */ + RESTORE_ARGS 0,8,0 + INTERRUPT_RETURN_NMI_SAFE + #ifdef CONFIG_PREEMPT /* Returning to kernel space. Check if we need preemption */ /* rcx: threadinfo. interrupts off. */ @@ -911,9 +933,17 @@ paranoid_swapgs\trace: TRACE_IRQS_IRETQ 0 .endif SWAPGS_UNSAFE_STACK -paranoid_restore\trace: +paranoid_restore_no_nmi\trace: RESTORE_ALL 8 jmp irq_return +paranoid_restore\trace: + GET_THREAD_INFO(%rcx) + testl $HARDNMI_MASK,threadinfo_preempt_count(%rcx) + jz paranoid_restore_no_nmi\trace /* Nested over NMI ? */ + testw $X86_EFLAGS_TF,EFLAGS-0(%rsp) /* trap flag? */ + jnz paranoid_restore_no_nmi\trace + RESTORE_ALL 8 + INTERRUPT_RETURN_NMI_SAFE paranoid_userspace\trace: GET_THREAD_INFO(%rcx) movl threadinfo_flags(%rcx),%ebx @@ -1012,7 +1042,7 @@ error_exit: TRACE_IRQS_OFF GET_THREAD_INFO(%rcx) testl %eax,%eax - jne retint_kernel + jne retexc_kernel LOCKDEP_SYS_EXIT_IRQ movl threadinfo_flags(%rcx),%edx movl $_TIF_WORK_MASK,%edi diff --git a/arch/x86/kernel/head64.c b/arch/x86/kernel/head64.c index e25c57b8aa84..4bcb61cd9fcd 100644 --- a/arch/x86/kernel/head64.c +++ b/arch/x86/kernel/head64.c @@ -25,6 +25,20 @@ #include <asm/e820.h> #include <asm/bios_ebda.h> +/* boot cpu pda */ +static struct x8664_pda _boot_cpu_pda __read_mostly; + +#ifdef CONFIG_SMP +/* + * We install an empty cpu_pda pointer table to indicate to early users + * (numa_set_node) that the cpu_pda pointer table for cpus other than + * the boot cpu is not yet setup. + */ +static struct x8664_pda *__cpu_pda[NR_CPUS] __initdata; +#else +static struct x8664_pda *__cpu_pda[NR_CPUS] __read_mostly; +#endif + static void __init zap_identity_mappings(void) { pgd_t *pgd = pgd_offset_k(0UL); @@ -156,10 +170,12 @@ void __init x86_64_start_kernel(char * real_mode_data) early_printk("Kernel alive\n"); - for (i = 0; i < NR_CPUS; i++) - cpu_pda(i) = &boot_cpu_pda[i]; - + _cpu_pda = __cpu_pda; + cpu_pda(0) = &_boot_cpu_pda; pda_init(0); + + early_printk("Kernel really alive\n"); + copy_bootdata(__va(real_mode_data)); reserve_early(__pa_symbol(&_text), __pa_symbol(&_end), "TEXT DATA BSS"); diff --git a/arch/x86/kernel/immediate.c b/arch/x86/kernel/immediate.c new file mode 100644 index 000000000000..30dcf9d0fe4d --- /dev/null +++ b/arch/x86/kernel/immediate.c @@ -0,0 +1,545 @@ +/* + * Immediate Value - x86 architecture specific code. + * + * Rationale + * + * Required because of : + * - Erratum 49 fix for Intel PIII. + * - Still present on newer processors : Intel Core 2 Duo Processor for Intel + * Centrino Duo Processor Technology Specification Update, AH33. + * Unsynchronized Cross-Modifying Code Operations Can Cause Unexpected + * Instruction Execution Results. + * + * Permits immediate value modification by XMC with correct serialization. + * + * Reentrant for NMI and trap handler instrumentation. Permits XMC to a + * location that has preemption enabled because it involves no temporary or + * reused data structure. + * + * Quoting Richard J Moore, source of the information motivating this + * implementation which differs from the one proposed by Intel which is not + * suitable for kernel context (does not support NMI and would require disabling + * interrupts on every CPU for a long period) : + * + * "There is another issue to consider when looking into using probes other + * then int3: + * + * Intel erratum 54 - Unsynchronized Cross-modifying code - refers to the + * practice of modifying code on one processor where another has prefetched + * the unmodified version of the code. Intel states that unpredictable general + * protection faults may result if a synchronizing instruction (iret, int, + * int3, cpuid, etc ) is not executed on the second processor before it + * executes the pre-fetched out-of-date copy of the instruction. + * + * When we became aware of this I had a long discussion with Intel's + * microarchitecture guys. It turns out that the reason for this erratum + * (which incidentally Intel does not intend to fix) is because the trace + * cache - the stream of micro-ops resulting from instruction interpretation - + * cannot be guaranteed to be valid. Reading between the lines I assume this + * issue arises because of optimization done in the trace cache, where it is + * no longer possible to identify the original instruction boundaries. If the + * CPU discoverers that the trace cache has been invalidated because of + * unsynchronized cross-modification then instruction execution will be + * aborted with a GPF. Further discussion with Intel revealed that replacing + * the first opcode byte with an int3 would not be subject to this erratum. + * + * So, is cmpxchg reliable? One has to guarantee more than mere atomicity." + * + * Overall design + * + * The algorithm proposed by Intel applies not so well in kernel context: it + * would imply disabling interrupts and looping on every CPUs while modifying + * the code and would not support instrumentation of code called from interrupt + * sources that cannot be disabled. + * + * Therefore, we use a different algorithm to respect Intel's erratum (see the + * quoted discussion above). We make sure that no CPU sees an out-of-date copy + * of a pre-fetched instruction by 1 - using a breakpoint, which skips the + * instruction that is going to be modified, 2 - issuing an IPI to every CPU to + * execute a sync_core(), to make sure that even when the breakpoint is removed, + * no cpu could possibly still have the out-of-date copy of the instruction, + * modify the now unused 2nd byte of the instruction, and then put back the + * original 1st byte of the instruction. + * + * It has exactly the same intent as the algorithm proposed by Intel, but + * it has less side-effects, scales better and supports NMI, SMI and MCE. + * + * Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca> + */ + +#include <linux/preempt.h> +#include <linux/smp.h> +#include <linux/notifier.h> +#include <linux/module.h> +#include <linux/immediate.h> +#include <linux/kdebug.h> +#include <linux/rcupdate.h> +#include <linux/kprobes.h> +#include <linux/io.h> + +#include <asm/cacheflush.h> + +#define BREAKPOINT_INSTRUCTION 0xcc +#define JMP_REL32 0xe9 +#define BREAKPOINT_INS_LEN 1 +#define NR_NOPS 10 + +/*#define DEBUG_IMMEDIATE 1*/ + +#ifdef DEBUG_IMMEDIATE +#define printk_dbg printk +#else +#define printk_dbg(fmt , a...) +#endif + +static unsigned long target_after_int3; /* EIP of the target after the int3 */ +static unsigned long bypass_eip; /* EIP of the bypass. */ +static unsigned long bypass_after_int3; /* EIP after the end-of-bypass int3 */ +static unsigned long after_imv; /* + * EIP where to resume after the + * single-stepping. + */ + +/* + * Internal bypass used during value update. The bypass is skipped by the + * function in which it is inserted. + * No need to be aligned because we exclude readers from the site during + * update. + * Layout is: + * (10x nop) int3 + * (maximum size is 2 bytes opcode + 8 bytes immediate value for long on x86_64) + * The nops are the target replaced by the instruction to single-step. + * Align on 16 bytes to make sure the nops fit within a single page so remapping + * it can be done easily. + */ +static inline void _imv_bypass(unsigned long *bypassaddr, + unsigned long *breaknextaddr) +{ + asm volatile("jmp 2f;\n\t" + ".align 16;\n\t" + "0:\n\t" + ".space 10, 0x90;\n\t" + "1:\n\t" + "int3;\n\t" + "2:\n\t" + "mov $(0b),%0;\n\t" + "mov $((1b)+1),%1;\n\t" + : "=r" (*bypassaddr), + "=r" (*breaknextaddr)); +} + +static void imv_synchronize_core(void *info) +{ + sync_core(); /* use cpuid to stop speculative execution */ +} + +/* + * The eip value points right after the breakpoint instruction, in the second + * byte of the movl. + * Disable preemption in the bypass to make sure no thread will be preempted in + * it. We can then use synchronize_sched() to make sure every bypass users have + * ended. + */ +static int imv_notifier(struct notifier_block *nb, + unsigned long val, void *data) +{ + enum die_val die_val = (enum die_val) val; + struct die_args *args = data; + + if (!args->regs || user_mode_vm(args->regs)) + return NOTIFY_DONE; + + if (die_val == DIE_INT3) { + if (args->regs->ip == target_after_int3) { + /* deal with non-relocatable jmp instructions */ + switch (*(uint8_t *)bypass_eip) { + case JMP_REL32: /* e9 cw jmp rel16 (valid on ia32) */ + /* e9 cd jmp rel32 */ + /* Skip the 4 bytes (jump offset) + * following the breakpoint. */ + args->regs->ip += + *(int *)(bypass_eip + 1) + 4; + /* The bypass won't be executed. */ + return NOTIFY_STOP; + } + preempt_disable(); + args->regs->ip = bypass_eip; + return NOTIFY_STOP; + } else if (args->regs->ip == bypass_after_int3) { + args->regs->ip = after_imv; + preempt_enable(); + return NOTIFY_STOP; + } + } + return NOTIFY_DONE; +} + +static struct notifier_block imv_notify = { + .notifier_call = imv_notifier, + .priority = 0x7fffffff, /* we need to be notified first */ +}; + +/* + * returns -1 if not found + * return 0 if found. + */ +static inline int detect_mov_test_jne(uint8_t *addr, uint8_t **opcode, + uint8_t **jmp_offset, int *offset_len) +{ + uint8_t *addr1, *addr2; + + printk_dbg(KERN_DEBUG "Trying at %p %hx %hx %hx %hx %hx %hx\n", + addr, addr[0], addr[1], addr[2], addr[3], addr[4], addr[5]); + /* + * b8 imm32 mov imm32,%eax or + * c9 XX XX XX XX jmp rel32 (patched instruction) + */ + if (addr[0] != 0xb8 && addr[0] != JMP_REL32) + return -1; + /* 85 c0 test %eax,%eax */ + if (addr[5] != 0x85 || addr[6] != 0xc0) + return -1; + printk_dbg(KERN_DEBUG "Found test %%eax,%%eax at %p\n", addr + 5); + switch (addr[7]) { + case 0x75: /* 75 cb jne rel8 */ + printk_dbg(KERN_DEBUG "Found jne rel8 at %p\n", addr + 7); + *opcode = addr + 7; + *jmp_offset = addr + 8; + *offset_len = 1; + /* addr1 is the address following the branch instruction */ + addr1 = addr + 9; + /* addr2 is the branch target */ + addr2 = addr1 + addr[8]; + break; + case 0x0f: + switch (addr[8]) { + case 0x85: /* 0F 85 cw jne rel16 (valid on ia32) */ + /* 0F 85 cd jne rel32 */ + printk_dbg(KERN_DEBUG "Found jne rel16/32 at %p\n", + addr + 8); + *opcode = addr + 7; + *jmp_offset = addr + 9; + *offset_len = 4; + /* + * addr1 is the address following the branch + * instruction + */ + addr1 = addr + 13; + /* addr2 is the branch target */ + addr2 = addr1 + *(uint32_t *)(addr + 9); + break; + default: + return -1; + } + break; + default: return -1; + } + /* + * Now check that the pattern was squeezed between the mov instruction + * end the two epilogues (branch taken and not taken), which ensure + * %eax and ZF liveliness is limited to our instructions. + */ + if (!is_imv_cond_end((unsigned long)addr1, (unsigned long)addr2)) + return -1; + return 0; +} + +/* + * returns -1 if not found + * return 0 if found. + */ +static inline int detect_mov_test_je(uint8_t *addr, uint8_t **opcode, + uint8_t **jmp_offset, int *offset_len) +{ + uint8_t *addr1, *addr2; + + /* + * b8 imm32 mov imm32,%eax or + * c9 XX XX XX XX jmp rel32 (patched instruction) + */ + if (addr[0] != 0xb8 && addr[0] != JMP_REL32) + return -1; + /* 85 c0 test %eax,%eax */ + if (addr[5] != 0x85 || addr[6] != 0xc0) + return -1; + printk_dbg(KERN_DEBUG "Found test %%eax,%%eax at %p\n", addr + 5); + switch (addr[7]) { + case 0x74: /* 74 cb je rel8 */ + printk_dbg(KERN_DEBUG "Found je rel8 at %p\n", addr + 7); + *opcode = addr + 7; + *jmp_offset = addr + 8; + *offset_len = 1; + /* addr1 is the address following the branch instruction */ + addr1 = addr + 9; + /* addr2 is the branch target */ + addr2 = addr1 + addr[8]; + break; + case 0x0f: + switch (addr[8]) { + case 0x84: /* 0F 84 cw je rel16 (valid on ia32) */ + /* 0F 84 cd je rel32 */ + printk_dbg(KERN_DEBUG "Found je rel16/32 at %p\n", + addr + 8); + *opcode = addr + 7; + *jmp_offset = addr + 9; + *offset_len = 4; + /* + * addr1 is the address following the branch + * instruction + */ + addr1 = addr + 13; + /* addr2 is the branch target */ + addr2 = addr1 + *(uint32_t *)(addr + 9); + break; + default: + return -1; + } + break; + default: return -1; + } + /* + * Now check that the pattern was squeezed between the mov instruction + * end the two epilogues (branch taken and not taken), which ensure + * %eax and ZF liveliness is limited to our instructions. + */ + if (!is_imv_cond_end((unsigned long)addr1, (unsigned long)addr2)) + return -1; + return 0; +} + +static int static_early; + +/* + * Marked noinline because we prefer to have only one _imv_bypass. Not that it + * is required, but there is no need to edit two bypasses. + */ +static noinline int replace_instruction_safe(uint8_t *addr, uint8_t *newcode, + int size) +{ + char *vaddr; + struct page *pages[1]; + int len; + int ret; + + /* bypass is 10 bytes long for x86_64 long */ + WARN_ON(size > 10); + + _imv_bypass(&bypass_eip, &bypass_after_int3); + + if (!static_early) { + after_imv = (unsigned long)addr + size; + + /* + * Using the _early variants because nobody is executing the + * bypass code while we patch it. It is protected by the + * imv_mutex. Since we modify the instructions non atomically + * (for nops), we have to use the _early variant. + * We must however deal with RO pages. + * Use a single page : 10 bytes are aligned on 16 bytes + * boundaries. + */ + pages[0] = virt_to_page((void *)bypass_eip); + vaddr = vmap(pages, 1, VM_MAP, PAGE_KERNEL); + BUG_ON(!vaddr); + text_poke_early(&vaddr[bypass_eip & ~PAGE_MASK], + (void *)addr, size); + /* + * Fill the rest with nops. + */ + len = NR_NOPS - size; + add_nops((void *) + &vaddr[(bypass_eip & ~PAGE_MASK) + size], + len); + vunmap(vaddr); + + target_after_int3 = (unsigned long)addr + BREAKPOINT_INS_LEN; + /* register_die_notifier has memory barriers */ + register_die_notifier(&imv_notify); + /* The breakpoint will execute the bypass */ + text_poke((void *)addr, + ((unsigned char[]){BREAKPOINT_INSTRUCTION}), + BREAKPOINT_INS_LEN); + /* + * Make sure the breakpoint is set before we continue (visible + * to other CPUs and interrupts). + */ + wmb(); + /* + * Execute serializing instruction on each CPU. + */ + ret = on_each_cpu(imv_synchronize_core, NULL, 1, 1); + BUG_ON(ret != 0); + + text_poke((void *)(addr + BREAKPOINT_INS_LEN), + &newcode[BREAKPOINT_INS_LEN], + size - BREAKPOINT_INS_LEN); + /* + * Make sure the value can be seen from other CPUs and + * interrupts. + */ + wmb(); +#ifdef DEBUG_IMMEDIATE + mdelay(10); /* lets the breakpoint for a while */ +#endif + text_poke(addr, newcode, BREAKPOINT_INS_LEN); + /* + * Wait for all int3 handlers to end (interrupts are disabled in + * int3). This CPU is clearly not in a int3 handler, because + * int3 handler is not preemptible and there cannot be any more + * int3 handler called for this site, because we placed the + * original instruction back. synchronize_sched has memory + * barriers. + */ + synchronize_sched(); + unregister_die_notifier(&imv_notify); + /* unregister_die_notifier has memory barriers */ + } else + text_poke_early(addr, newcode, size); + return 0; +} + +static int patch_jump_target(struct __imv *imv) +{ + uint8_t *opcode, *jmp_offset; + int offset_len; + int mov_test_j_found = 0; + unsigned long logicvar; + + if (!detect_mov_test_jne((uint8_t *)imv->imv - 1, + &opcode, &jmp_offset, &offset_len)) { + logicvar = imv->var; /* positive logic */ + mov_test_j_found = 1; + } else if (!detect_mov_test_je((uint8_t *)imv->imv - 1, + &opcode, &jmp_offset, &offset_len)) { + logicvar = !imv->var; /* negative logic */ + mov_test_j_found = 1; + } + + if (mov_test_j_found) { + int newoff; + + if (offset_len == 1) { + if (logicvar) + newoff = *(signed char *)jmp_offset; + else + newoff = 0; + newoff += 4; /* jump over test, branch */ + } else { + if (logicvar) + newoff = *(int *)jmp_offset; + else + newoff = 0; + newoff += 8; /* + * jump over test (2 bytes) + * and branch (6 bytes). + */ + } + /* Speed up by skipping if not changed */ + if (*(uint8_t *)(imv->imv - 1) == JMP_REL32 && + *(int *)imv->imv == newoff) + return 0; + /* replace with a 5 bytes jump */ + replace_instruction_safe((uint8_t *)imv->imv - 1, + ((unsigned char[]){ JMP_REL32, + ((unsigned char *)&newoff)[0], + ((unsigned char *)&newoff)[1], + ((unsigned char *)&newoff)[2], + ((unsigned char *)&newoff)[3] }), + 5); + return 0; + } + /* Nothing known found. */ + return -1; +} + +/** + * arch_imv_update - update one immediate value + * @imv: pointer of type const struct __imv to update + * @early: early boot (1) or normal (0) + * + * Update one immediate value. Must be called with imv_mutex held. + */ +__kprobes int arch_imv_update(struct __imv *imv, int early) +{ + int ret; + uint8_t buf[10]; + unsigned long insn, opcode_size; + + static_early = early; + + /* + * If imv_cond is encountered, try to patch it with + * patch_jump_target. Continue with normal immediate values if the area + * surrounding the instruction is not as expected. + */ + if (imv->size == 0) { + ret = patch_jump_target(imv); + if (ret) { +#ifdef DEBUG_IMMEDIATE + static int nr_fail; + printk(KERN_DEBUG + "Jump target fallback at %lX, nr fail %d\n", + imv->imv, ++nr_fail); +#endif + imv->size = 4; + } else { +#ifdef DEBUG_IMMEDIATE + static int nr_success; + printk(KERN_DEBUG "Jump target at %lX, nr success %d\n", + imv->imv, ++nr_success); +#endif + return 0; + } + } + + opcode_size = imv->insn_size - imv->size; + insn = imv->imv - opcode_size; + +#ifdef CONFIG_KPROBES + /* + * Fail if a kprobe has been set on this instruction. + * (TODO: we could eventually do better and modify all the (possibly + * nested) kprobes for this site if kprobes had an API for this. + */ + if (unlikely(!early + && *(unsigned char *)insn == BREAKPOINT_INSTRUCTION)) { + printk(KERN_WARNING "Immediate value in conflict with kprobe. " + "Variable at %p, " + "instruction at %p, size %hu\n", + (void *)imv->var, + (void *)imv->imv, imv->size); + return -EBUSY; + } +#endif + + /* + * If the variable and the instruction have the same value, there is + * nothing to do. + */ + BUG_ON(imv->var_size > imv->size); + switch (imv->var_size) { + case 1: if (*(uint8_t *)imv->imv == *(uint8_t *)imv->var) + return 0; + break; + case 2: if (*(uint16_t *)imv->imv == *(uint16_t *)imv->var) + return 0; + break; + case 4: if (*(uint32_t *)imv->imv == *(uint32_t *)imv->var) + return 0; + break; +#ifdef CONFIG_X86_64 + case 8: if (*(uint64_t *)imv->imv == *(uint64_t *)imv->var) + return 0; + break; +#endif + default:return -EINVAL; + } + + memcpy(buf, (uint8_t *)insn, opcode_size); + memcpy(&buf[opcode_size], (void *)imv->var, imv->var_size); + /* pad MSBs with 0 */ + memset(&buf[opcode_size + imv->var_size], 0, imv->size - imv->var_size); + replace_instruction_safe((uint8_t *)insn, buf, imv->insn_size); + + return 0; +} diff --git a/arch/x86/kernel/io_apic_64.c b/arch/x86/kernel/io_apic_64.c index 0081cbd3d01b..7f7eacd1d8fe 100644 --- a/arch/x86/kernel/io_apic_64.c +++ b/arch/x86/kernel/io_apic_64.c @@ -722,7 +722,7 @@ static int __assign_irq_vector(int irq, cpumask_t mask) return 0; } - for_each_cpu_mask(cpu, mask) { + for_each_cpu_mask_nr(cpu, mask) { cpumask_t domain, new_mask; int new_cpu; int vector, offset; @@ -743,7 +743,7 @@ next: continue; if (vector == IA32_SYSCALL_VECTOR) goto next; - for_each_cpu_mask(new_cpu, new_mask) + for_each_cpu_mask_nr(new_cpu, new_mask) if (per_cpu(vector_irq, new_cpu)[vector] != -1) goto next; /* Found one! */ @@ -753,7 +753,7 @@ next: cfg->move_in_progress = 1; cfg->old_domain = cfg->domain; } - for_each_cpu_mask(new_cpu, new_mask) + for_each_cpu_mask_nr(new_cpu, new_mask) per_cpu(vector_irq, new_cpu)[vector] = irq; cfg->vector = vector; cfg->domain = domain; @@ -785,7 +785,7 @@ static void __clear_irq_vector(int irq) vector = cfg->vector; cpus_and(mask, cfg->domain, cpu_online_map); - for_each_cpu_mask(cpu, mask) + for_each_cpu_mask_nr(cpu, mask) per_cpu(vector_irq, cpu)[vector] = -1; cfg->vector = 0; diff --git a/arch/x86/kernel/kprobes.c b/arch/x86/kernel/kprobes.c index b8c6743a13da..82b7fd318356 100644 --- a/arch/x86/kernel/kprobes.c +++ b/arch/x86/kernel/kprobes.c @@ -376,9 +376,7 @@ void __kprobes arch_disarm_kprobe(struct kprobe *p) void __kprobes arch_remove_kprobe(struct kprobe *p) { - mutex_lock(&kprobe_mutex); free_insn_slot(p->ainsn.insn, (p->ainsn.boostable == 1)); - mutex_unlock(&kprobe_mutex); } static void __kprobes save_previous_kprobe(struct kprobe_ctlblk *kcb) diff --git a/arch/x86/kernel/nmi_64.c b/arch/x86/kernel/nmi_64.c index 33cb8ecbb6dd..deaf2d2f3280 100644 --- a/arch/x86/kernel/nmi_64.c +++ b/arch/x86/kernel/nmi_64.c @@ -88,7 +88,7 @@ int __init check_nmi_watchdog(void) if (!atomic_read(&nmi_active)) return 0; - prev_nmi_count = kmalloc(NR_CPUS * sizeof(int), GFP_KERNEL); + prev_nmi_count = kmalloc(nr_cpu_ids * sizeof(int), GFP_KERNEL); if (!prev_nmi_count) return -1; @@ -99,7 +99,7 @@ int __init check_nmi_watchdog(void) smp_call_function(nmi_cpu_busy, (void *)&endflag, 0, 0); #endif - for (cpu = 0; cpu < NR_CPUS; cpu++) + for (cpu = 0; cpu < nr_cpu_ids; cpu++) prev_nmi_count[cpu] = cpu_pda(cpu)->__nmi_count; local_irq_enable(); mdelay((20*1000)/nmi_hz); // wait 20 ticks diff --git a/arch/x86/kernel/paravirt.c b/arch/x86/kernel/paravirt.c index 74f0c5ea2a03..bb174a80ac8a 100644 --- a/arch/x86/kernel/paravirt.c +++ b/arch/x86/kernel/paravirt.c @@ -139,6 +139,7 @@ unsigned paravirt_patch_default(u8 type, u16 clobbers, void *insnbuf, /* If the operation is a nop, then nop the callsite */ ret = paravirt_patch_nop(); else if (type == PARAVIRT_PATCH(pv_cpu_ops.iret) || + type == PARAVIRT_PATCH(pv_cpu_ops.nmi_return) || type == PARAVIRT_PATCH(pv_cpu_ops.irq_enable_syscall_ret)) /* If operation requires a jmp, then jmp */ ret = paravirt_patch_jmp(insnbuf, opfunc, addr, len); @@ -190,6 +191,7 @@ static void native_flush_tlb_single(unsigned long addr) /* These are in entry.S */ extern void native_iret(void); +extern void native_nmi_return(void); extern void native_irq_enable_syscall_ret(void); static int __init print_banner(void) @@ -328,6 +330,7 @@ struct pv_cpu_ops pv_cpu_ops = { .irq_enable_syscall_ret = native_irq_enable_syscall_ret, .iret = native_iret, + .nmi_return = native_nmi_return, .swapgs = native_swapgs, .set_iopl_mask = native_set_iopl_mask, diff --git a/arch/x86/kernel/paravirt_patch_32.c b/arch/x86/kernel/paravirt_patch_32.c index 82fc5fcab4f4..8ed31c7755a5 100644 --- a/arch/x86/kernel/paravirt_patch_32.c +++ b/arch/x86/kernel/paravirt_patch_32.c @@ -1,10 +1,13 @@ -#include <asm/paravirt.h> +#include <linux/stringify.h> +#include <linux/irqflags.h> DEF_NATIVE(pv_irq_ops, irq_disable, "cli"); DEF_NATIVE(pv_irq_ops, irq_enable, "sti"); DEF_NATIVE(pv_irq_ops, restore_fl, "push %eax; popf"); DEF_NATIVE(pv_irq_ops, save_fl, "pushf; pop %eax"); DEF_NATIVE(pv_cpu_ops, iret, "iret"); +DEF_NATIVE(pv_cpu_ops, nmi_return, + __stringify(NATIVE_INTERRUPT_RETURN_NMI_SAFE)); DEF_NATIVE(pv_cpu_ops, irq_enable_syscall_ret, "sti; sysexit"); DEF_NATIVE(pv_mmu_ops, read_cr2, "mov %cr2, %eax"); DEF_NATIVE(pv_mmu_ops, write_cr3, "mov %eax, %cr3"); @@ -29,6 +32,7 @@ unsigned native_patch(u8 type, u16 clobbers, void *ibuf, PATCH_SITE(pv_irq_ops, restore_fl); PATCH_SITE(pv_irq_ops, save_fl); PATCH_SITE(pv_cpu_ops, iret); + PATCH_SITE(pv_cpu_ops, nmi_return); PATCH_SITE(pv_cpu_ops, irq_enable_syscall_ret); PATCH_SITE(pv_mmu_ops, read_cr2); PATCH_SITE(pv_mmu_ops, read_cr3); diff --git a/arch/x86/kernel/paravirt_patch_64.c b/arch/x86/kernel/paravirt_patch_64.c index 7d904e138d7e..56eccea8c731 100644 --- a/arch/x86/kernel/paravirt_patch_64.c +++ b/arch/x86/kernel/paravirt_patch_64.c @@ -1,12 +1,15 @@ +#include <linux/irqflags.h> +#include <linux/stringify.h> #include <asm/paravirt.h> #include <asm/asm-offsets.h> -#include <linux/stringify.h> DEF_NATIVE(pv_irq_ops, irq_disable, "cli"); DEF_NATIVE(pv_irq_ops, irq_enable, "sti"); DEF_NATIVE(pv_irq_ops, restore_fl, "pushq %rdi; popfq"); DEF_NATIVE(pv_irq_ops, save_fl, "pushfq; popq %rax"); DEF_NATIVE(pv_cpu_ops, iret, "iretq"); +DEF_NATIVE(pv_cpu_ops, nmi_return, + __stringify(NATIVE_INTERRUPT_RETURN_NMI_SAFE)); DEF_NATIVE(pv_mmu_ops, read_cr2, "movq %cr2, %rax"); DEF_NATIVE(pv_mmu_ops, read_cr3, "movq %cr3, %rax"); DEF_NATIVE(pv_mmu_ops, write_cr3, "movq %rdi, %cr3"); @@ -35,6 +38,7 @@ unsigned native_patch(u8 type, u16 clobbers, void *ibuf, PATCH_SITE(pv_irq_ops, irq_enable); PATCH_SITE(pv_irq_ops, irq_disable); PATCH_SITE(pv_cpu_ops, iret); + PATCH_SITE(pv_cpu_ops, nmi_return); PATCH_SITE(pv_cpu_ops, irq_enable_syscall_ret); PATCH_SITE(pv_cpu_ops, swapgs); PATCH_SITE(pv_mmu_ops, read_cr2); diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c index c0c68c18a788..4f450b668183 100644 --- a/arch/x86/kernel/setup.c +++ b/arch/x86/kernel/setup.c @@ -14,17 +14,39 @@ unsigned int num_processors; unsigned disabled_cpus __cpuinitdata; + /* Processor that is doing the boot up */ unsigned int boot_cpu_physical_apicid = -1U; EXPORT_SYMBOL(boot_cpu_physical_apicid); -DEFINE_PER_CPU(u16, x86_cpu_to_apicid) = BAD_APICID; -EXPORT_PER_CPU_SYMBOL(x86_cpu_to_apicid); - /* Bitmask of physically existing CPUs */ physid_mask_t phys_cpu_present_map; -#if defined(CONFIG_HAVE_SETUP_PER_CPU_AREA) && defined(CONFIG_SMP) +/* map cpu index to physical APIC ID */ +DEFINE_EARLY_PER_CPU(u16, x86_cpu_to_apicid, BAD_APICID); +DEFINE_EARLY_PER_CPU(u16, x86_bios_cpu_apicid, BAD_APICID); +EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_apicid); +EXPORT_EARLY_PER_CPU_SYMBOL(x86_bios_cpu_apicid); + +#if defined(CONFIG_NUMA) && defined(CONFIG_X86_64) +#define X86_64_NUMA 1 + +/* map cpu index to node index */ +DEFINE_EARLY_PER_CPU(int, x86_cpu_to_node_map, NUMA_NO_NODE); +EXPORT_EARLY_PER_CPU_SYMBOL(x86_cpu_to_node_map); + +/* which logical CPUs are on which nodes */ +cpumask_t *node_to_cpumask_map; +EXPORT_SYMBOL(node_to_cpumask_map); + +/* setup node_to_cpumask_map */ +static void __init setup_node_to_cpumask_map(void); + +#else +static inline void setup_node_to_cpumask_map(void) { } +#endif + +#ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA /* * Copy data used in early init routines from the initial arrays to the * per cpu data areas. These arrays then become expendable and the @@ -35,20 +57,21 @@ static void __init setup_per_cpu_maps(void) int cpu; for_each_possible_cpu(cpu) { - per_cpu(x86_cpu_to_apicid, cpu) = x86_cpu_to_apicid_init[cpu]; + per_cpu(x86_cpu_to_apicid, cpu) = + early_per_cpu_map(x86_cpu_to_apicid, cpu); per_cpu(x86_bios_cpu_apicid, cpu) = - x86_bios_cpu_apicid_init[cpu]; -#ifdef CONFIG_NUMA + early_per_cpu_map(x86_bios_cpu_apicid, cpu); +#ifdef X86_64_NUMA per_cpu(x86_cpu_to_node_map, cpu) = - x86_cpu_to_node_map_init[cpu]; + early_per_cpu_map(x86_cpu_to_node_map, cpu); #endif } /* indicate the early static arrays will soon be gone */ - x86_cpu_to_apicid_early_ptr = NULL; - x86_bios_cpu_apicid_early_ptr = NULL; -#ifdef CONFIG_NUMA - x86_cpu_to_node_map_early_ptr = NULL; + early_per_cpu_ptr(x86_cpu_to_apicid) = NULL; + early_per_cpu_ptr(x86_bios_cpu_apicid) = NULL; +#ifdef X86_64_NUMA + early_per_cpu_ptr(x86_cpu_to_node_map) = NULL; #endif } @@ -77,6 +100,50 @@ static inline void setup_cpumask_of_cpu(void) { } */ unsigned long __per_cpu_offset[NR_CPUS] __read_mostly; EXPORT_SYMBOL(__per_cpu_offset); +static inline void setup_cpu_pda_map(void) { } + +#elif !defined(CONFIG_SMP) +static inline void setup_cpu_pda_map(void) { } + +#else /* CONFIG_SMP && CONFIG_X86_64 */ + +/* + * Allocate cpu_pda pointer table and array via alloc_bootmem. + */ +static void __init setup_cpu_pda_map(void) +{ + char *pda; + struct x8664_pda **new_cpu_pda; + unsigned long size; + int cpu; + + size = roundup(sizeof(struct x8664_pda), cache_line_size()); + + /* allocate cpu_pda array and pointer table */ + { + unsigned long tsize = nr_cpu_ids * sizeof(void *); + unsigned long asize = size * (nr_cpu_ids - 1); + + tsize = roundup(tsize, cache_line_size()); + new_cpu_pda = alloc_bootmem(tsize + asize); + pda = (char *)new_cpu_pda + tsize; + } + + /* initialize pointer table to static pda's */ + for_each_possible_cpu(cpu) { + if (cpu == 0) { + /* leave boot cpu pda in place */ + new_cpu_pda[0] = cpu_pda(0); + continue; + } + new_cpu_pda[cpu] = (struct x8664_pda *)pda; + new_cpu_pda[cpu]->in_bootmem = 1; + pda += size; + } + + /* point to new pointer table */ + _cpu_pda = new_cpu_pda; +} #endif /* @@ -86,52 +153,225 @@ EXPORT_SYMBOL(__per_cpu_offset); */ void __init setup_per_cpu_areas(void) { - int i, highest_cpu = 0; - unsigned long size; + ssize_t size = PERCPU_ENOUGH_ROOM; + char *ptr; + int cpu; #ifdef CONFIG_HOTPLUG_CPU prefill_possible_map(); +#else + nr_cpu_ids = num_processors; #endif + /* Setup cpu_pda map */ + setup_cpu_pda_map(); + /* Copy section for each CPU (we discard the original) */ size = PERCPU_ENOUGH_ROOM; printk(KERN_INFO "PERCPU: Allocating %lu bytes of per cpu data\n", size); - for_each_possible_cpu(i) { - char *ptr; + for_each_possible_cpu(cpu) { #ifndef CONFIG_NEED_MULTIPLE_NODES ptr = alloc_bootmem_pages(size); #else - int node = early_cpu_to_node(i); + int node = early_cpu_to_node(cpu); if (!node_online(node) || !NODE_DATA(node)) { ptr = alloc_bootmem_pages(size); printk(KERN_INFO - "cpu %d has no node or node-local memory\n", i); + "cpu %d has no node %d or node-local memory\n", + cpu, node); } else ptr = alloc_bootmem_pages_node(NODE_DATA(node), size); #endif - if (!ptr) - panic("Cannot allocate cpu data for CPU %d\n", i); -#ifdef CONFIG_X86_64 - cpu_pda(i)->data_offset = ptr - __per_cpu_start; -#else - __per_cpu_offset[i] = ptr - __per_cpu_start; -#endif + per_cpu_offset(cpu) = ptr - __per_cpu_start; memcpy(ptr, __per_cpu_start, __per_cpu_end - __per_cpu_start); - highest_cpu = i; } - nr_cpu_ids = highest_cpu + 1; - printk(KERN_DEBUG "NR_CPUS: %d, nr_cpu_ids: %d\n", NR_CPUS, nr_cpu_ids); + printk(KERN_DEBUG "NR_CPUS: %d, nr_cpu_ids: %d, nr_node_ids %d\n", + NR_CPUS, nr_cpu_ids, nr_node_ids); /* Setup percpu data maps */ setup_per_cpu_maps(); + /* Setup node to cpumask map */ + setup_node_to_cpumask_map(); + /* Setup cpumask_of_cpu map */ setup_cpumask_of_cpu(); } #endif + +#ifdef X86_64_NUMA + +/* + * Allocate node_to_cpumask_map based on number of available nodes + * Requires node_possible_map to be valid. + * + * Note: node_to_cpumask() is not valid until after this is done. + */ +static void __init setup_node_to_cpumask_map(void) +{ + unsigned int node, num = 0; + cpumask_t *map; + + /* setup nr_node_ids if not done yet */ + if (nr_node_ids == MAX_NUMNODES) { + for_each_node_mask(node, node_possible_map) + num = node; + nr_node_ids = num + 1; + } + + /* allocate the map */ + map = alloc_bootmem_low(nr_node_ids * sizeof(cpumask_t)); + + Dprintk(KERN_DEBUG "Node to cpumask map at %p for %d nodes\n", + map, nr_node_ids); + + /* node_to_cpumask() will now work */ + node_to_cpumask_map = map; +} + +void __cpuinit numa_set_node(int cpu, int node) +{ + int *cpu_to_node_map = early_per_cpu_ptr(x86_cpu_to_node_map); + + if (cpu_pda(cpu) && node != NUMA_NO_NODE) + cpu_pda(cpu)->nodenumber = node; + + if (cpu_to_node_map) + cpu_to_node_map[cpu] = node; + + else if (per_cpu_offset(cpu)) + per_cpu(x86_cpu_to_node_map, cpu) = node; + + else + Dprintk(KERN_INFO "Setting node for non-present cpu %d\n", cpu); +} + +void __cpuinit numa_clear_node(int cpu) +{ + numa_set_node(cpu, NUMA_NO_NODE); +} + +#ifndef CONFIG_DEBUG_PER_CPU_MAPS + +void __cpuinit numa_add_cpu(int cpu) +{ + cpu_set(cpu, node_to_cpumask_map[early_cpu_to_node(cpu)]); +} + +void __cpuinit numa_remove_cpu(int cpu) +{ + cpu_clear(cpu, node_to_cpumask_map[cpu_to_node(cpu)]); +} + +#else /* CONFIG_DEBUG_PER_CPU_MAPS */ + +/* + * --------- debug versions of the numa functions --------- + */ +static void __cpuinit numa_set_cpumask(int cpu, int enable) +{ + int node = cpu_to_node(cpu); + cpumask_t *mask; + char buf[64]; + + if (node_to_cpumask_map == NULL) { + printk(KERN_ERR "node_to_cpumask_map NULL\n"); + dump_stack(); + return; + } + + mask = &node_to_cpumask_map[node]; + if (enable) + cpu_set(cpu, *mask); + else + cpu_clear(cpu, *mask); + + cpulist_scnprintf(buf, sizeof(buf), *mask); + printk(KERN_DEBUG "%s cpu %d node %d: mask now %s\n", + enable? "numa_add_cpu":"numa_remove_cpu", cpu, node, buf); + } + +void __cpuinit numa_add_cpu(int cpu) +{ + numa_set_cpumask(cpu, 1); +} + +void __cpuinit numa_remove_cpu(int cpu) +{ + numa_set_cpumask(cpu, 0); +} + +int cpu_to_node(int cpu) +{ + if (early_per_cpu_ptr(x86_cpu_to_node_map)) { + printk(KERN_WARNING + "cpu_to_node(%d): usage too early!\n", cpu); + dump_stack(); + return early_per_cpu_ptr(x86_cpu_to_node_map)[cpu]; + } + return per_cpu(x86_cpu_to_node_map, cpu); +} +EXPORT_SYMBOL(cpu_to_node); + +/* + * Same function as cpu_to_node() but used if called before the + * per_cpu areas are setup. + */ +int early_cpu_to_node(int cpu) +{ + if (early_per_cpu_ptr(x86_cpu_to_node_map)) + return early_per_cpu_ptr(x86_cpu_to_node_map)[cpu]; + + if (!per_cpu_offset(cpu)) { + printk(KERN_WARNING + "early_cpu_to_node(%d): no per_cpu area!\n", cpu); + dump_stack(); + return NUMA_NO_NODE; + } + return per_cpu(x86_cpu_to_node_map, cpu); +} + +/* + * Returns a pointer to the bitmask of CPUs on Node 'node'. + */ +cpumask_t *_node_to_cpumask_ptr(int node) +{ + if (node_to_cpumask_map == NULL) { + printk(KERN_WARNING + "_node_to_cpumask_ptr(%d): no node_to_cpumask_map!\n", + node); + dump_stack(); + return &cpu_online_map; + } + return &node_to_cpumask_map[node]; +} +EXPORT_SYMBOL(_node_to_cpumask_ptr); + +/* + * Returns a bitmask of CPUs on Node 'node'. + */ +cpumask_t node_to_cpumask(int node) +{ + if (node_to_cpumask_map == NULL) { + printk(KERN_WARNING + "node_to_cpumask(%d): no node_to_cpumask_map!\n", node); + dump_stack(); + return cpu_online_map; + } + return node_to_cpumask_map[node]; +} +EXPORT_SYMBOL(node_to_cpumask); + +/* + * --------- end of debug versions of the numa functions --------- + */ + +#endif /* CONFIG_DEBUG_PER_CPU_MAPS */ + +#endif /* X86_64_NUMA */ diff --git a/arch/x86/kernel/setup64.c b/arch/x86/kernel/setup64.c index aee0e8200777..631ea6cc01d8 100644 --- a/arch/x86/kernel/setup64.c +++ b/arch/x86/kernel/setup64.c @@ -12,6 +12,7 @@ #include <linux/bitops.h> #include <linux/module.h> #include <linux/kgdb.h> +#include <linux/topology.h> #include <asm/pda.h> #include <asm/pgtable.h> #include <asm/processor.h> @@ -34,9 +35,8 @@ struct boot_params boot_params; cpumask_t cpu_initialized __cpuinitdata = CPU_MASK_NONE; -struct x8664_pda *_cpu_pda[NR_CPUS] __read_mostly; +struct x8664_pda **_cpu_pda __read_mostly; EXPORT_SYMBOL(_cpu_pda); -struct x8664_pda boot_cpu_pda[NR_CPUS] __cacheline_aligned; struct desc_ptr idt_descr = { 256 * 16 - 1, (unsigned long) idt_table }; @@ -114,8 +114,10 @@ void pda_init(int cpu) __get_free_pages(GFP_ATOMIC, IRQSTACK_ORDER); if (!pda->irqstackptr) panic("cannot allocate irqstack for cpu %d", cpu); - } + if (pda->nodenumber == 0 && cpu_to_node(cpu) != NUMA_NO_NODE) + pda->nodenumber = cpu_to_node(cpu); + } pda->irqstackptr += IRQSTACKSIZE-64; } diff --git a/arch/x86/kernel/setup_32.c b/arch/x86/kernel/setup_32.c index b54c79c91efd..2e6622da5373 100644 --- a/arch/x86/kernel/setup_32.c +++ b/arch/x86/kernel/setup_32.c @@ -689,18 +689,6 @@ static void set_mca_bus(int x) static void set_mca_bus(int x) { } #endif -#ifdef CONFIG_NUMA -/* - * In the golden day, when everything among i386 and x86_64 will be - * integrated, this will not live here - */ -void *x86_cpu_to_node_map_early_ptr; -int x86_cpu_to_node_map_init[NR_CPUS] = { - [0 ... NR_CPUS-1] = NUMA_NO_NODE -}; -DEFINE_PER_CPU(int, x86_cpu_to_node_map) = NUMA_NO_NODE; -#endif - /* * Determine if we were loaded by an EFI loader. If so, then we have also been * passed the efi memmap, systab, etc., so we should use these data structures @@ -835,18 +823,6 @@ void __init setup_arch(char **cmdline_p) io_delay_init(); -#ifdef CONFIG_X86_SMP - /* - * setup to use the early static init tables during kernel startup - * X86_SMP will exclude sub-arches that don't deal well with it. - */ - x86_cpu_to_apicid_early_ptr = (void *)x86_cpu_to_apicid_init; - x86_bios_cpu_apicid_early_ptr = (void *)x86_bios_cpu_apicid_init; -#ifdef CONFIG_NUMA - x86_cpu_to_node_map_early_ptr = (void *)x86_cpu_to_node_map_init; -#endif -#endif - #ifdef CONFIG_X86_GENERICARCH generic_apic_probe(); #endif diff --git a/arch/x86/kernel/setup_64.c b/arch/x86/kernel/setup_64.c index bee7a7195aa0..1f2a55e19ccc 100644 --- a/arch/x86/kernel/setup_64.c +++ b/arch/x86/kernel/setup_64.c @@ -394,15 +394,6 @@ void __init setup_arch(char **cmdline_p) kvmclock_init(); #endif -#ifdef CONFIG_SMP - /* setup to use the early static init tables during kernel startup */ - x86_cpu_to_apicid_early_ptr = (void *)x86_cpu_to_apicid_init; - x86_bios_cpu_apicid_early_ptr = (void *)x86_bios_cpu_apicid_init; -#ifdef CONFIG_NUMA - x86_cpu_to_node_map_early_ptr = (void *)x86_cpu_to_node_map_init; -#endif -#endif - #ifdef CONFIG_ACPI /* * Initialize the ACPI boot-time table parser (gets the RSDP and SDT). diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c index 7b9a7645b600..a6c7267cb8a8 100644 --- a/arch/x86/kernel/smpboot.c +++ b/arch/x86/kernel/smpboot.c @@ -68,22 +68,6 @@ #include <mach_wakecpu.h> #include <smpboot_hooks.h> -/* - * FIXME: For x86_64, those are defined in other files. But moving them here, - * would make the setup areas dependent on smp, which is a loss. When we - * integrate apic between arches, we can probably do a better job, but - * right now, they'll stay here -- glommer - */ - -/* which logical CPU number maps to which CPU (physical APIC ID) */ -u16 x86_cpu_to_apicid_init[NR_CPUS] __initdata = - { [0 ... NR_CPUS-1] = BAD_APICID }; -void *x86_cpu_to_apicid_early_ptr; - -u16 x86_bios_cpu_apicid_init[NR_CPUS] __initdata - = { [0 ... NR_CPUS-1] = BAD_APICID }; -void *x86_bios_cpu_apicid_early_ptr; - #ifdef CONFIG_X86_32 u8 apicid_2_node[MAX_APICID]; #endif @@ -480,7 +464,7 @@ void __cpuinit set_cpu_sibling_map(int cpu) cpu_set(cpu, cpu_sibling_setup_map); if (smp_num_siblings > 1) { - for_each_cpu_mask(i, cpu_sibling_setup_map) { + for_each_cpu_mask_nr(i, cpu_sibling_setup_map) { if (c->phys_proc_id == cpu_data(i).phys_proc_id && c->cpu_core_id == cpu_data(i).cpu_core_id) { cpu_set(i, per_cpu(cpu_sibling_map, cpu)); @@ -503,7 +487,7 @@ void __cpuinit set_cpu_sibling_map(int cpu) return; } - for_each_cpu_mask(i, cpu_sibling_setup_map) { + for_each_cpu_mask_nr(i, cpu_sibling_setup_map) { if (per_cpu(cpu_llc_id, cpu) != BAD_APICID && per_cpu(cpu_llc_id, cpu) == per_cpu(cpu_llc_id, i)) { cpu_set(i, c->llc_shared_map); @@ -825,6 +809,43 @@ static void __cpuinit do_fork_idle(struct work_struct *work) complete(&c_idle->done); } +/* + * Allocate node local memory for the AP pda. + * + * Must be called after the _cpu_pda pointer table is initialized. + */ +static int __cpuinit get_local_pda(int cpu) +{ + struct x8664_pda *oldpda, *newpda; + unsigned long size = sizeof(struct x8664_pda); + int node = cpu_to_node(cpu); + + if (cpu_pda(cpu) && !cpu_pda(cpu)->in_bootmem) + return 0; + + oldpda = cpu_pda(cpu); + newpda = kmalloc_node(size, GFP_ATOMIC, node); + if (!newpda) { + printk(KERN_ERR "Could not allocate node local PDA " + "for CPU %d on node %d\n", cpu, node); + + if (oldpda) + return 0; /* have a usable pda */ + else + return -1; + } + + if (oldpda) { + memcpy(newpda, oldpda, size); + if (!after_bootmem) + free_bootmem((unsigned long)oldpda, size); + } + + newpda->in_bootmem = 0; + cpu_pda(cpu) = newpda; + return 0; +} + static int __cpuinit do_boot_cpu(int apicid, int cpu) /* * NOTE - on most systems this is a PHYSICAL apic ID, but on multiquad @@ -850,19 +871,11 @@ static int __cpuinit do_boot_cpu(int apicid, int cpu) } /* Allocate node local memory for AP pdas */ - if (cpu_pda(cpu) == &boot_cpu_pda[cpu]) { - struct x8664_pda *newpda, *pda; - int node = cpu_to_node(cpu); - pda = cpu_pda(cpu); - newpda = kmalloc_node(sizeof(struct x8664_pda), GFP_ATOMIC, - node); - if (newpda) { - memcpy(newpda, pda, sizeof(struct x8664_pda)); - cpu_pda(cpu) = newpda; - } else - printk(KERN_ERR - "Could not allocate node local PDA for CPU %d on node %d\n", - cpu, node); + if (cpu > 0) { + boot_error = get_local_pda(cpu); + if (boot_error) + goto restore_state; + /* if can't get pda memory, can't start cpu */ } #endif @@ -981,11 +994,13 @@ do_rest: } } +restore_state: + if (boot_error) { /* Try to put things back the way they were before ... */ unmap_cpu_to_logical_apicid(cpu); #ifdef CONFIG_X86_64 - clear_node_cpumask(cpu); /* was set by numa_add_cpu */ + numa_remove_cpu(cpu); /* was set by numa_add_cpu */ #endif cpu_clear(cpu, cpu_callout_map); /* was set by do_boot_cpu() */ cpu_clear(cpu, cpu_initialized); /* was set by cpu_init() */ @@ -1291,7 +1306,7 @@ static void remove_siblinginfo(int cpu) int sibling; struct cpuinfo_x86 *c = &cpu_data(cpu); - for_each_cpu_mask(sibling, per_cpu(cpu_core_map, cpu)) { + for_each_cpu_mask_nr(sibling, per_cpu(cpu_core_map, cpu)) { cpu_clear(cpu, per_cpu(cpu_core_map, sibling)); /*/ * last thread sibling in this cpu core going down @@ -1300,7 +1315,7 @@ static void remove_siblinginfo(int cpu) cpu_data(sibling).booted_cores--; } - for_each_cpu_mask(sibling, per_cpu(cpu_sibling_map, cpu)) + for_each_cpu_mask_nr(sibling, per_cpu(cpu_sibling_map, cpu)) cpu_clear(cpu, per_cpu(cpu_sibling_map, sibling)); cpus_clear(per_cpu(cpu_sibling_map, cpu)); cpus_clear(per_cpu(cpu_core_map, cpu)); @@ -1354,6 +1369,8 @@ __init void prefill_possible_map(void) for (i = 0; i < possible; i++) cpu_set(i, cpu_possible_map); + + nr_cpu_ids = possible; } static void __ref remove_cpu_from_maps(int cpu) @@ -1364,7 +1381,7 @@ static void __ref remove_cpu_from_maps(int cpu) cpu_clear(cpu, cpu_callin_map); /* was set by cpu_init() */ clear_bit(cpu, (unsigned long *)&cpu_initialized); - clear_node_cpumask(cpu); + numa_remove_cpu(cpu); #endif } diff --git a/arch/x86/kernel/traps_32.c b/arch/x86/kernel/traps_32.c index 7fad08484460..39acc4a247e9 100644 --- a/arch/x86/kernel/traps_32.c +++ b/arch/x86/kernel/traps_32.c @@ -476,6 +476,9 @@ void die(const char *str, struct pt_regs *regs, long err) if (kexec_should_crash(current)) crash_kexec(regs); + if (in_nmi()) + panic("Fatal exception in non-maskable interrupt"); + if (in_interrupt()) panic("Fatal exception in interrupt"); @@ -592,7 +595,7 @@ void do_##name(struct pt_regs *regs, long error_code) \ } DO_VM86_ERROR_INFO(0, SIGFPE, "divide error", divide_error, FPE_INTDIV, regs->ip) -#ifndef CONFIG_KPROBES +#if !defined(CONFIG_KPROBES) && !defined(CONFIG_IMMEDIATE) DO_VM86_ERROR(3, SIGTRAP, "int3", int3) #endif DO_VM86_ERROR(4, SIGSEGV, "overflow", overflow) @@ -857,7 +860,7 @@ void restart_nmi(void) acpi_nmi_enable(); } -#ifdef CONFIG_KPROBES +#if defined(CONFIG_KPROBES) || defined(CONFIG_IMMEDIATE) void __kprobes do_int3(struct pt_regs *regs, long error_code) { trace_hardirqs_fixup(); @@ -866,8 +869,8 @@ void __kprobes do_int3(struct pt_regs *regs, long error_code) == NOTIFY_STOP) return; /* - * This is an interrupt gate, because kprobes wants interrupts - * disabled. Normal trap handlers don't. + * This is an interrupt gate, because kprobes and immediate values want + * interrupts disabled. Normal trap handlers don't. */ restore_interrupts(regs); diff --git a/arch/x86/kernel/traps_64.c b/arch/x86/kernel/traps_64.c index a2749e5bb49e..8c3cf3e8306d 100644 --- a/arch/x86/kernel/traps_64.c +++ b/arch/x86/kernel/traps_64.c @@ -555,6 +555,10 @@ void __kprobes oops_end(unsigned long flags, struct pt_regs *regs, int signr) oops_exit(); return; } + if (in_nmi()) + panic("Fatal exception in non-maskable interrupt"); + if (in_interrupt()) + panic("Fatal exception in interrupt"); if (panic_on_oops) panic("Fatal exception"); oops_exit(); diff --git a/arch/x86/kernel/vmi_32.c b/arch/x86/kernel/vmi_32.c index 956f38927aa7..01d687d69a76 100644 --- a/arch/x86/kernel/vmi_32.c +++ b/arch/x86/kernel/vmi_32.c @@ -151,6 +151,8 @@ static unsigned vmi_patch(u8 type, u16 clobbers, void *insns, insns, ip); case PARAVIRT_PATCH(pv_cpu_ops.iret): return patch_internal(VMI_CALL_IRET, len, insns, ip); + case PARAVIRT_PATCH(pv_cpu_ops.nmi_return): + return patch_internal(VMI_CALL_IRET, len, insns, ip); case PARAVIRT_PATCH(pv_cpu_ops.irq_enable_syscall_ret): return patch_internal(VMI_CALL_SYSEXIT, len, insns, ip); default: diff --git a/arch/x86/kernel/vmlinux_32.lds.S b/arch/x86/kernel/vmlinux_32.lds.S index 248a575da05f..1e85602e8cfa 100644 --- a/arch/x86/kernel/vmlinux_32.lds.S +++ b/arch/x86/kernel/vmlinux_32.lds.S @@ -204,6 +204,7 @@ SECTIONS /* Sections to be discarded */ /DISCARD/ : { *(.exitcall.exit) + *(__discard) } STABS_DEBUG diff --git a/arch/x86/kernel/vmlinux_64.lds.S b/arch/x86/kernel/vmlinux_64.lds.S index 2f8cd2987271..1d92d83d1e03 100644 --- a/arch/x86/kernel/vmlinux_64.lds.S +++ b/arch/x86/kernel/vmlinux_64.lds.S @@ -231,6 +231,7 @@ SECTIONS /DISCARD/ : { *(.exitcall.exit) *(.eh_frame) + *(__discard) } STABS_DEBUG diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c index 979f9838c5e4..adba164894f8 100644 --- a/arch/x86/kvm/x86.c +++ b/arch/x86/kvm/x86.c @@ -2869,7 +2869,7 @@ again: /* * Profile KVM exit RIPs: */ - if (unlikely(prof_on == KVM_PROFILING)) { + if (unlikely(imv_read(prof_on) == KVM_PROFILING)) { kvm_x86_ops->cache_regs(vcpu); profile_hit(KVM_PROFILING, (void *)vcpu->arch.rip); } diff --git a/arch/x86/lguest/boot.c b/arch/x86/lguest/boot.c index af65b2da3ba0..f5cbb74d9f2d 100644 --- a/arch/x86/lguest/boot.c +++ b/arch/x86/lguest/boot.c @@ -958,6 +958,7 @@ __init void lguest_init(void) pv_cpu_ops.cpuid = lguest_cpuid; pv_cpu_ops.load_idt = lguest_load_idt; pv_cpu_ops.iret = lguest_iret; + pv_cpu_ops.nmi_return = lguest_iret; pv_cpu_ops.load_sp0 = lguest_load_sp0; pv_cpu_ops.load_tr_desc = lguest_load_tr_desc; pv_cpu_ops.set_ldt = lguest_set_ldt; diff --git a/arch/x86/mm/fault.c b/arch/x86/mm/fault.c index 4c3dcf553f3d..af8e5a05152b 100644 --- a/arch/x86/mm/fault.c +++ b/arch/x86/mm/fault.c @@ -540,6 +540,7 @@ static int vmalloc_fault(struct pt_regs *regs, unsigned long address, } return 0; #else + unsigned long pgd_paddr; pgd_t *pgd, *pgd_ref; pud_t *pud, *pud_ref; pmd_t *pmd, *pmd_ref; @@ -553,7 +554,8 @@ static int vmalloc_fault(struct pt_regs *regs, unsigned long address, happen within a race in page table update. In the later case just flush. */ - pgd = pgd_offset(current->mm ?: &init_mm, address); + pgd_paddr = read_cr3(); + pgd = __va(pgd_paddr) + pgd_index(address); pgd_ref = pgd_offset_k(address); if (pgd_none(*pgd_ref)) return -1; diff --git a/arch/x86/mm/kmmio.c b/arch/x86/mm/kmmio.c index 93b1797666cb..93d82038af4b 100644 --- a/arch/x86/mm/kmmio.c +++ b/arch/x86/mm/kmmio.c @@ -6,6 +6,7 @@ */ #include <linux/list.h> +#include <linux/rculist.h> #include <linux/spinlock.h> #include <linux/hash.h> #include <linux/init.h> @@ -104,11 +105,12 @@ static struct kmmio_fault_page *get_kmmio_fault_page(unsigned long page) return NULL; } -static void set_page_present(unsigned long addr, bool present, int *pglevel) +static void set_page_present(unsigned long addr, bool present, + unsigned int *pglevel) { pteval_t pteval; pmdval_t pmdval; - int level; + unsigned int level; pmd_t *pmd; pte_t *pte = lookup_address(addr, &level); @@ -145,15 +147,15 @@ static void set_page_present(unsigned long addr, bool present, int *pglevel) } /** Mark the given page as not present. Access to it will trigger a fault. */ -static void arm_kmmio_fault_page(unsigned long page, int *page_level) +static void arm_kmmio_fault_page(unsigned long page, unsigned int *pglevel) { - set_page_present(page & PAGE_MASK, false, page_level); + set_page_present(page & PAGE_MASK, false, pglevel); } /** Mark the given page as present. */ -static void disarm_kmmio_fault_page(unsigned long page, int *page_level) +static void disarm_kmmio_fault_page(unsigned long page, unsigned int *pglevel) { - set_page_present(page & PAGE_MASK, true, page_level); + set_page_present(page & PAGE_MASK, true, pglevel); } /* diff --git a/arch/x86/mm/mmio-mod.c b/arch/x86/mm/mmio-mod.c index ed0e0e90b3ef..e7397e108beb 100644 --- a/arch/x86/mm/mmio-mod.c +++ b/arch/x86/mm/mmio-mod.c @@ -137,7 +137,7 @@ static ssize_t write_marker(struct file *file, const char __user *buffer, static void print_pte(unsigned long address) { - int level; + unsigned int level; pte_t *pte = lookup_address(address, &level); if (!pte) { diff --git a/arch/x86/mm/numa_64.c b/arch/x86/mm/numa_64.c index 425894e3e793..a1f3778b4680 100644 --- a/arch/x86/mm/numa_64.c +++ b/arch/x86/mm/numa_64.c @@ -31,23 +31,10 @@ static bootmem_data_t plat_node_bdata[MAX_NUMNODES]; struct memnode memnode; -#ifdef CONFIG_SMP -int x86_cpu_to_node_map_init[NR_CPUS] = { - [0 ... NR_CPUS-1] = NUMA_NO_NODE -}; -void *x86_cpu_to_node_map_early_ptr; -EXPORT_SYMBOL(x86_cpu_to_node_map_early_ptr); -#endif -DEFINE_PER_CPU(int, x86_cpu_to_node_map) = NUMA_NO_NODE; -EXPORT_PER_CPU_SYMBOL(x86_cpu_to_node_map); - s16 apicid_to_node[MAX_LOCAL_APIC] __cpuinitdata = { [0 ... MAX_LOCAL_APIC-1] = NUMA_NO_NODE }; -cpumask_t node_to_cpumask_map[MAX_NUMNODES] __read_mostly; -EXPORT_SYMBOL(node_to_cpumask_map); - int numa_off __initdata; static unsigned long __initdata nodemap_addr; static unsigned long __initdata nodemap_size; @@ -570,31 +557,10 @@ void __init numa_initmem_init(unsigned long start_pfn, unsigned long last_pfn) node_set(0, node_possible_map); for (i = 0; i < NR_CPUS; i++) numa_set_node(i, 0); - /* cpumask_of_cpu() may not be available during early startup */ - memset(&node_to_cpumask_map[0], 0, sizeof(node_to_cpumask_map[0])); - cpu_set(0, node_to_cpumask_map[0]); e820_register_active_regions(0, start_pfn, last_pfn); setup_node_bootmem(0, start_pfn << PAGE_SHIFT, last_pfn << PAGE_SHIFT); } -__cpuinit void numa_add_cpu(int cpu) -{ - set_bit(cpu, - (unsigned long *)&node_to_cpumask_map[early_cpu_to_node(cpu)]); -} - -void __cpuinit numa_set_node(int cpu, int node) -{ - int *cpu_to_node_map = x86_cpu_to_node_map_early_ptr; - - if(cpu_to_node_map) - cpu_to_node_map[cpu] = node; - else if(per_cpu_offset(cpu)) - per_cpu(x86_cpu_to_node_map, cpu) = node; - else - Dprintk(KERN_INFO "Setting node for non-present cpu %d\n", cpu); -} - unsigned long __init numa_free_all_bootmem(void) { unsigned long pages = 0; @@ -641,6 +607,7 @@ static __init int numa_setup(char *opt) } early_param("numa", numa_setup); +#ifdef CONFIG_NUMA /* * Setup early cpu_to_node. * @@ -652,14 +619,19 @@ early_param("numa", numa_setup); * is already initialized in a round robin manner at numa_init_array, * prior to this call, and this initialization is good enough * for the fake NUMA cases. + * + * Called before the per_cpu areas are setup. */ void __init init_cpu_to_node(void) { - int i; + int cpu; + u16 *cpu_to_apicid = early_per_cpu_ptr(x86_cpu_to_apicid); - for (i = 0; i < NR_CPUS; i++) { + BUG_ON(cpu_to_apicid == NULL); + + for_each_possible_cpu(cpu) { int node; - u16 apicid = x86_cpu_to_apicid_init[i]; + u16 apicid = cpu_to_apicid[cpu]; if (apicid == BAD_APICID) continue; @@ -668,8 +640,9 @@ void __init init_cpu_to_node(void) continue; if (!node_online(node)) continue; - numa_set_node(i, node); + numa_set_node(cpu, node); } } +#endif diff --git a/arch/x86/mm/srat_64.c b/arch/x86/mm/srat_64.c index f6d0584cbf38..1766170069c9 100644 --- a/arch/x86/mm/srat_64.c +++ b/arch/x86/mm/srat_64.c @@ -403,7 +403,7 @@ int __init acpi_scan_nodes(unsigned long start, unsigned long end) if (node == NUMA_NO_NODE) continue; if (!node_isset(node, node_possible_map)) - numa_set_node(i, NUMA_NO_NODE); + numa_clear_node(i); } numa_init_array(); return 0; diff --git a/arch/x86/xen/enlighten.c b/arch/x86/xen/enlighten.c index c8a56e457d61..33272ce6f189 100644 --- a/arch/x86/xen/enlighten.c +++ b/arch/x86/xen/enlighten.c @@ -1008,6 +1008,7 @@ static const struct pv_cpu_ops xen_cpu_ops __initdata = { .read_pmc = native_read_pmc, .iret = xen_iret, + .nmi_return = xen_iret, .irq_enable_syscall_ret = xen_sysexit, .load_tr_desc = paravirt_nop, diff --git a/arch/x86/xen/smp.c b/arch/x86/xen/smp.c index 94e69000f982..7a70638797ed 100644 --- a/arch/x86/xen/smp.c +++ b/arch/x86/xen/smp.c @@ -345,7 +345,7 @@ static void xen_send_IPI_mask(cpumask_t mask, enum ipi_vector vector) cpus_and(mask, mask, cpu_online_map); - for_each_cpu_mask(cpu, mask) + for_each_cpu_mask_nr(cpu, mask) xen_send_IPI_one(cpu, vector); } @@ -413,7 +413,7 @@ int xen_smp_call_function_mask(cpumask_t mask, void (*func)(void *), /* Make sure other vcpus get a chance to run if they need to. */ yield = false; - for_each_cpu_mask(cpu, mask) + for_each_cpu_mask_nr(cpu, mask) if (xen_vcpu_stolen(cpu)) yield = true; diff --git a/crypto/async_tx/async_tx.c b/crypto/async_tx/async_tx.c index c6e772fc5ccd..095c798d3170 100644 --- a/crypto/async_tx/async_tx.c +++ b/crypto/async_tx/async_tx.c @@ -23,6 +23,7 @@ * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. * */ +#include <linux/rculist.h> #include <linux/kernel.h> #include <linux/async_tx.h> diff --git a/drivers/acpi/processor_throttling.c b/drivers/acpi/processor_throttling.c index bb06738860c4..28509fbba6f9 100644 --- a/drivers/acpi/processor_throttling.c +++ b/drivers/acpi/processor_throttling.c @@ -1013,7 +1013,7 @@ int acpi_processor_set_throttling(struct acpi_processor *pr, int state) * affected cpu in order to get one proper T-state. * The notifier event is THROTTLING_PRECHANGE. */ - for_each_cpu_mask(i, online_throttling_cpus) { + for_each_cpu_mask_nr(i, online_throttling_cpus) { t_state.cpu = i; acpi_processor_throttling_notifier(THROTTLING_PRECHANGE, &t_state); @@ -1034,7 +1034,7 @@ int acpi_processor_set_throttling(struct acpi_processor *pr, int state) * it is necessary to set T-state for every affected * cpus. */ - for_each_cpu_mask(i, online_throttling_cpus) { + for_each_cpu_mask_nr(i, online_throttling_cpus) { match_pr = processors[i]; /* * If the pointer is invalid, we will report the @@ -1068,7 +1068,7 @@ int acpi_processor_set_throttling(struct acpi_processor *pr, int state) * affected cpu to update the T-states. * The notifier event is THROTTLING_POSTCHANGE */ - for_each_cpu_mask(i, online_throttling_cpus) { + for_each_cpu_mask_nr(i, online_throttling_cpus) { t_state.cpu = i; acpi_processor_throttling_notifier(THROTTLING_POSTCHANGE, &t_state); diff --git a/drivers/base/cpu.c b/drivers/base/cpu.c index e38dfed41d80..509e1e4c0241 100644 --- a/drivers/base/cpu.c +++ b/drivers/base/cpu.c @@ -119,17 +119,19 @@ static ssize_t print_cpus_##type(struct sysdev_class *class, char *buf) \ { \ return print_cpus_map(buf, &cpu_##type##_map); \ } \ -struct sysdev_class_attribute attr_##type##_map = \ +static struct sysdev_class_attribute attr_##type##_map = \ _SYSDEV_CLASS_ATTR(type, 0444, print_cpus_##type, NULL) print_cpus_func(online); print_cpus_func(possible); print_cpus_func(present); +print_cpus_func(system); -struct sysdev_class_attribute *cpu_state_attr[] = { +static struct sysdev_class_attribute *cpu_state_attr[] = { &attr_online_map, &attr_possible_map, &attr_present_map, + &attr_system_map, }; static int cpu_states_init(void) diff --git a/drivers/base/topology.c b/drivers/base/topology.c index fdf4044d2e74..1efe162e16d7 100644 --- a/drivers/base/topology.c +++ b/drivers/base/topology.c @@ -40,6 +40,7 @@ static ssize_t show_##name(struct sys_device *dev, char *buf) \ return sprintf(buf, "%d\n", topology_##name(cpu)); \ } +#if defined(topology_thread_siblings) || defined(topology_core_siblings) static ssize_t show_cpumap(int type, cpumask_t *mask, char *buf) { ptrdiff_t len = PTR_ALIGN(buf + PAGE_SIZE - 1, PAGE_SIZE) - buf; @@ -54,21 +55,41 @@ static ssize_t show_cpumap(int type, cpumask_t *mask, char *buf) } return n; } +#endif +#ifdef arch_provides_topology_pointers #define define_siblings_show_map(name) \ -static inline ssize_t show_##name(struct sys_device *dev, char *buf) \ +static ssize_t show_##name(struct sys_device *dev, char *buf) \ { \ unsigned int cpu = dev->id; \ return show_cpumap(0, &(topology_##name(cpu)), buf); \ } #define define_siblings_show_list(name) \ -static inline ssize_t show_##name##_list(struct sys_device *dev, char *buf) \ +static ssize_t show_##name##_list(struct sys_device *dev, char *buf) \ { \ unsigned int cpu = dev->id; \ return show_cpumap(1, &(topology_##name(cpu)), buf); \ } +#else +#define define_siblings_show_map(name) \ +static ssize_t show_##name(struct sys_device *dev, char *buf) \ +{ \ + unsigned int cpu = dev->id; \ + cpumask_t mask = topology_##name(cpu); \ + return show_cpumap(0, &mask, buf); \ +} + +#define define_siblings_show_list(name) \ +static ssize_t show_##name##_list(struct sys_device *dev, char *buf) \ +{ \ + unsigned int cpu = dev->id; \ + cpumask_t mask = topology_##name(cpu); \ + return show_cpumap(1, &mask, buf); \ +} +#endif + #define define_siblings_show_func(name) \ define_siblings_show_map(name); define_siblings_show_list(name) diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c index 7fce038fa57e..4937f2f97d8c 100644 --- a/drivers/cpufreq/cpufreq.c +++ b/drivers/cpufreq/cpufreq.c @@ -589,7 +589,7 @@ static ssize_t show_cpus(cpumask_t mask, char *buf) ssize_t i = 0; unsigned int cpu; - for_each_cpu_mask(cpu, mask) { + for_each_cpu_mask_nr(cpu, mask) { if (i) i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), " "); i += scnprintf(&buf[i], (PAGE_SIZE - i - 2), "%u", cpu); @@ -835,7 +835,7 @@ static int cpufreq_add_dev(struct sys_device *sys_dev) } #endif - for_each_cpu_mask(j, policy->cpus) { + for_each_cpu_mask_nr(j, policy->cpus) { if (cpu == j) continue; @@ -898,14 +898,14 @@ static int cpufreq_add_dev(struct sys_device *sys_dev) } spin_lock_irqsave(&cpufreq_driver_lock, flags); - for_each_cpu_mask(j, policy->cpus) { + for_each_cpu_mask_nr(j, policy->cpus) { cpufreq_cpu_data[j] = policy; per_cpu(policy_cpu, j) = policy->cpu; } spin_unlock_irqrestore(&cpufreq_driver_lock, flags); /* symlink affected CPUs */ - for_each_cpu_mask(j, policy->cpus) { + for_each_cpu_mask_nr(j, policy->cpus) { if (j == cpu) continue; if (!cpu_online(j)) @@ -945,7 +945,7 @@ static int cpufreq_add_dev(struct sys_device *sys_dev) err_out_unregister: spin_lock_irqsave(&cpufreq_driver_lock, flags); - for_each_cpu_mask(j, policy->cpus) + for_each_cpu_mask_nr(j, policy->cpus) cpufreq_cpu_data[j] = NULL; spin_unlock_irqrestore(&cpufreq_driver_lock, flags); @@ -1028,7 +1028,7 @@ static int __cpufreq_remove_dev(struct sys_device *sys_dev) * links afterwards. */ if (unlikely(cpus_weight(data->cpus) > 1)) { - for_each_cpu_mask(j, data->cpus) { + for_each_cpu_mask_nr(j, data->cpus) { if (j == cpu) continue; cpufreq_cpu_data[j] = NULL; @@ -1038,7 +1038,7 @@ static int __cpufreq_remove_dev(struct sys_device *sys_dev) spin_unlock_irqrestore(&cpufreq_driver_lock, flags); if (unlikely(cpus_weight(data->cpus) > 1)) { - for_each_cpu_mask(j, data->cpus) { + for_each_cpu_mask_nr(j, data->cpus) { if (j == cpu) continue; dprintk("removing link for cpu %u\n", j); diff --git a/drivers/cpufreq/cpufreq_conservative.c b/drivers/cpufreq/cpufreq_conservative.c index 5d3a04ba6ad2..fe565ee43757 100644 --- a/drivers/cpufreq/cpufreq_conservative.c +++ b/drivers/cpufreq/cpufreq_conservative.c @@ -497,7 +497,7 @@ static int cpufreq_governor_dbs(struct cpufreq_policy *policy, return rc; } - for_each_cpu_mask(j, policy->cpus) { + for_each_cpu_mask_nr(j, policy->cpus) { struct cpu_dbs_info_s *j_dbs_info; j_dbs_info = &per_cpu(cpu_dbs_info, j); j_dbs_info->cur_policy = policy; diff --git a/drivers/cpufreq/cpufreq_ondemand.c b/drivers/cpufreq/cpufreq_ondemand.c index d2af20dda382..33855cb3cf16 100644 --- a/drivers/cpufreq/cpufreq_ondemand.c +++ b/drivers/cpufreq/cpufreq_ondemand.c @@ -367,7 +367,7 @@ static void dbs_check_cpu(struct cpu_dbs_info_s *this_dbs_info) /* Get Idle Time */ idle_ticks = UINT_MAX; - for_each_cpu_mask(j, policy->cpus) { + for_each_cpu_mask_nr(j, policy->cpus) { cputime64_t total_idle_ticks; unsigned int tmp_idle_ticks; struct cpu_dbs_info_s *j_dbs_info; @@ -521,7 +521,7 @@ static int cpufreq_governor_dbs(struct cpufreq_policy *policy, return rc; } - for_each_cpu_mask(j, policy->cpus) { + for_each_cpu_mask_nr(j, policy->cpus) { struct cpu_dbs_info_s *j_dbs_info; j_dbs_info = &per_cpu(cpu_dbs_info, j); j_dbs_info->cur_policy = policy; diff --git a/drivers/infiniband/hw/ehca/ehca_irq.c b/drivers/infiniband/hw/ehca/ehca_irq.c index ca5eb0cb628c..4489fe1ed6e0 100644 --- a/drivers/infiniband/hw/ehca/ehca_irq.c +++ b/drivers/infiniband/hw/ehca/ehca_irq.c @@ -637,8 +637,8 @@ static inline int find_next_online_cpu(struct ehca_comp_pool *pool) ehca_dmp(&cpu_online_map, sizeof(cpumask_t), ""); spin_lock_irqsave(&pool->last_cpu_lock, flags); - cpu = next_cpu(pool->last_cpu, cpu_online_map); - if (cpu == NR_CPUS) + cpu = next_cpu_nr(pool->last_cpu, cpu_online_map); + if (cpu >= nr_cpu_ids) cpu = first_cpu(cpu_online_map); pool->last_cpu = cpu; spin_unlock_irqrestore(&pool->last_cpu_lock, flags); diff --git a/drivers/infiniband/hw/ipath/ipath_verbs.c b/drivers/infiniband/hw/ipath/ipath_verbs.c index e63927cce5b5..f165f6577ce7 100644 --- a/drivers/infiniband/hw/ipath/ipath_verbs.c +++ b/drivers/infiniband/hw/ipath/ipath_verbs.c @@ -35,6 +35,7 @@ #include <rdma/ib_user_verbs.h> #include <linux/io.h> #include <linux/utsname.h> +#include <linux/rculist.h> #include "ipath_kernel.h" #include "ipath_verbs.h" diff --git a/drivers/infiniband/hw/ipath/ipath_verbs_mcast.c b/drivers/infiniband/hw/ipath/ipath_verbs_mcast.c index 9e5abf9c309d..d73e32232879 100644 --- a/drivers/infiniband/hw/ipath/ipath_verbs_mcast.c +++ b/drivers/infiniband/hw/ipath/ipath_verbs_mcast.c @@ -31,8 +31,7 @@ * SOFTWARE. */ -#include <linux/list.h> -#include <linux/rcupdate.h> +#include <linux/rculist.h> #include "ipath_verbs.h" diff --git a/drivers/net/macvlan.c b/drivers/net/macvlan.c index 2056cfc624dc..4ba75d302827 100644 --- a/drivers/net/macvlan.c +++ b/drivers/net/macvlan.c @@ -20,7 +20,7 @@ #include <linux/errno.h> #include <linux/slab.h> #include <linux/string.h> -#include <linux/list.h> +#include <linux/rculist.h> #include <linux/notifier.h> #include <linux/netdevice.h> #include <linux/etherdevice.h> diff --git a/drivers/pci/pci-driver.c b/drivers/pci/pci-driver.c index 72cf61ed8f96..e1637bd82b8e 100644 --- a/drivers/pci/pci-driver.c +++ b/drivers/pci/pci-driver.c @@ -181,7 +181,7 @@ static int pci_call_probe(struct pci_driver *drv, struct pci_dev *dev, any need to change it. */ struct mempolicy *oldpol; cpumask_t oldmask = current->cpus_allowed; - int node = pcibus_to_node(dev->bus); + int node = dev_to_node(&dev->dev); if (node >= 0) { node_to_cpumask_ptr(nodecpumask, node); diff --git a/fs/buffer.c b/fs/buffer.c index a073f3f4f013..1df338a0ecc7 100644 --- a/fs/buffer.c +++ b/fs/buffer.c @@ -41,6 +41,7 @@ #include <linux/bitops.h> #include <linux/mpage.h> #include <linux/bit_spinlock.h> +#include <linux/marker.h> static int fsync_buffers_list(spinlock_t *lock, struct list_head *list); @@ -89,7 +90,9 @@ void unlock_buffer(struct buffer_head *bh) */ void __wait_on_buffer(struct buffer_head * bh) { + trace_mark(fs_buffer_wait_start, "bh %p", bh); wait_on_bit(&bh->b_state, BH_Lock, sync_buffer, TASK_UNINTERRUPTIBLE); + trace_mark(fs_buffer_wait_end, "bh %p", bh); } static void diff --git a/fs/compat.c b/fs/compat.c index 332a869d2c53..070456335b10 100644 --- a/fs/compat.c +++ b/fs/compat.c @@ -51,6 +51,7 @@ #include <linux/poll.h> #include <linux/mm.h> #include <linux/eventpoll.h> +#include <linux/marker.h> #include <asm/uaccess.h> #include <asm/mmu_context.h> @@ -1402,6 +1403,7 @@ int compat_do_execve(char * filename, retval = search_binary_handler(bprm, regs); if (retval >= 0) { + trace_mark(fs_exec, "filename %s", filename); /* execve success */ security_bprm_free(bprm); acct_update_integrals(current); diff --git a/fs/exec.c b/fs/exec.c index aeaa9791d8be..6fe6afd0c907 100644 --- a/fs/exec.c +++ b/fs/exec.c @@ -52,6 +52,7 @@ #include <linux/tsacct_kern.h> #include <linux/cn_proc.h> #include <linux/audit.h> +#include <linux/marker.h> #include <asm/uaccess.h> #include <asm/mmu_context.h> @@ -1319,6 +1320,7 @@ int do_execve(char * filename, retval = search_binary_handler(bprm,regs); if (retval >= 0) { + trace_mark(fs_exec, "filename %s", filename); /* execve success */ free_arg_pages(bprm); security_bprm_free(bprm); diff --git a/fs/ioctl.c b/fs/ioctl.c index 7db32b3382d3..9abd3bda9bb3 100644 --- a/fs/ioctl.c +++ b/fs/ioctl.c @@ -13,6 +13,7 @@ #include <linux/security.h> #include <linux/module.h> #include <linux/uaccess.h> +#include <linux/marker.h> #include <asm/ioctls.h> @@ -201,6 +202,8 @@ asmlinkage long sys_ioctl(unsigned int fd, unsigned int cmd, unsigned long arg) if (!filp) goto out; + trace_mark(fs_ioctl, "fd %u cmd %u arg %lu", fd, cmd, arg); + error = security_file_ioctl(filp, cmd, arg); if (error) goto out_fput; diff --git a/fs/open.c b/fs/open.c index a1450086e92f..375407e860b8 100644 --- a/fs/open.c +++ b/fs/open.c @@ -28,6 +28,7 @@ #include <linux/rcupdate.h> #include <linux/audit.h> #include <linux/falloc.h> +#include <linux/marker.h> int vfs_statfs(struct dentry *dentry, struct kstatfs *buf) { @@ -1090,6 +1091,7 @@ long do_sys_open(int dfd, const char __user *filename, int flags, int mode) fsnotify_open(f->f_path.dentry); fd_install(fd, f); } + trace_mark(fs_open, "fd %d filename %s", fd, tmp); } putname(tmp); } @@ -1179,6 +1181,7 @@ asmlinkage long sys_close(unsigned int fd) filp = fdt->fd[fd]; if (!filp) goto out_unlock; + trace_mark(fs_close, "fd %u", fd); rcu_assign_pointer(fdt->fd[fd], NULL); FD_CLR(fd, fdt->close_on_exec); __put_unused_fd(files, fd); diff --git a/fs/read_write.c b/fs/read_write.c index f0d1240a5c69..8e710e0a214e 100644 --- a/fs/read_write.c +++ b/fs/read_write.c @@ -16,6 +16,7 @@ #include <linux/syscalls.h> #include <linux/pagemap.h> #include <linux/splice.h> +#include <linux/marker.h> #include "read_write.h" #include <asm/uaccess.h> @@ -146,6 +147,9 @@ asmlinkage off_t sys_lseek(unsigned int fd, off_t offset, unsigned int origin) if (res != (loff_t)retval) retval = -EOVERFLOW; /* LFS: should only happen on 32 bit platforms */ } + + trace_mark(fs_lseek, "fd %u offset %ld origin %u", fd, offset, origin); + fput_light(file, fput_needed); bad: return retval; @@ -173,6 +177,10 @@ asmlinkage long sys_llseek(unsigned int fd, unsigned long offset_high, offset = vfs_llseek(file, ((loff_t) offset_high << 32) | offset_low, origin); + trace_mark(fs_llseek, "fd %u offset %llu origin %u", fd, + (unsigned long long)offset, + origin); + retval = (int)offset; if (offset >= 0) { retval = -EFAULT; @@ -359,6 +367,7 @@ asmlinkage ssize_t sys_read(unsigned int fd, char __user * buf, size_t count) file = fget_light(fd, &fput_needed); if (file) { loff_t pos = file_pos_read(file); + trace_mark(fs_read, "fd %u count %zu", fd, count); ret = vfs_read(file, buf, count, &pos); file_pos_write(file, pos); fput_light(file, fput_needed); @@ -376,6 +385,7 @@ asmlinkage ssize_t sys_write(unsigned int fd, const char __user * buf, size_t co file = fget_light(fd, &fput_needed); if (file) { loff_t pos = file_pos_read(file); + trace_mark(fs_write, "fd %u count %zu", fd, count); ret = vfs_write(file, buf, count, &pos); file_pos_write(file, pos); fput_light(file, fput_needed); @@ -397,8 +407,12 @@ asmlinkage ssize_t sys_pread64(unsigned int fd, char __user *buf, file = fget_light(fd, &fput_needed); if (file) { ret = -ESPIPE; - if (file->f_mode & FMODE_PREAD) + if (file->f_mode & FMODE_PREAD) { + trace_mark(fs_pread64, "fd %u count %zu pos %llu", + fd, count, (unsigned long long)pos); ret = vfs_read(file, buf, count, &pos); + } + fput_light(file, fput_needed); } @@ -418,8 +432,11 @@ asmlinkage ssize_t sys_pwrite64(unsigned int fd, const char __user *buf, file = fget_light(fd, &fput_needed); if (file) { ret = -ESPIPE; - if (file->f_mode & FMODE_PWRITE) + if (file->f_mode & FMODE_PWRITE) { + trace_mark(fs_pwrite64, "fd %u count %zu pos %llu", + fd, count, (unsigned long long)pos); ret = vfs_write(file, buf, count, &pos); + } fput_light(file, fput_needed); } @@ -663,6 +680,7 @@ sys_readv(unsigned long fd, const struct iovec __user *vec, unsigned long vlen) file = fget_light(fd, &fput_needed); if (file) { loff_t pos = file_pos_read(file); + trace_mark(fs_readv, "fd %lu vlen %lu", fd, vlen); ret = vfs_readv(file, vec, vlen, &pos); file_pos_write(file, pos); fput_light(file, fput_needed); @@ -684,6 +702,7 @@ sys_writev(unsigned long fd, const struct iovec __user *vec, unsigned long vlen) file = fget_light(fd, &fput_needed); if (file) { loff_t pos = file_pos_read(file); + trace_mark(fs_writev, "fd %lu vlen %lu", fd, vlen); ret = vfs_writev(file, vec, vlen, &pos); file_pos_write(file, pos); fput_light(file, fput_needed); diff --git a/fs/select.c b/fs/select.c index 8dda969614a9..e2ba96dee70b 100644 --- a/fs/select.c +++ b/fs/select.c @@ -24,6 +24,7 @@ #include <linux/fdtable.h> #include <linux/fs.h> #include <linux/rcupdate.h> +#include <linux/marker.h> #include <asm/uaccess.h> @@ -232,6 +233,9 @@ int do_select(int n, fd_set_bits *fds, s64 *timeout) file = fget_light(i, &fput_needed); if (file) { f_op = file->f_op; + trace_mark(fs_select, + "fd %d timeout #8d%lld", + i, (long long)*timeout); mask = DEFAULT_POLLMASK; if (f_op && f_op->poll) mask = (*f_op->poll)(file, retval ? NULL : wait); @@ -560,6 +564,7 @@ static inline unsigned int do_pollfd(struct pollfd *pollfd, poll_table *pwait) file = fget_light(fd, &fput_needed); mask = POLLNVAL; if (file != NULL) { + trace_mark(fs_pollfd, "fd %d", fd); mask = DEFAULT_POLLMASK; if (file->f_op && file->f_op->poll) mask = file->f_op->poll(file, pwait); diff --git a/include/asm-alpha/thread_info.h b/include/asm-alpha/thread_info.h index fb3185196298..fde5f56cd770 100644 --- a/include/asm-alpha/thread_info.h +++ b/include/asm-alpha/thread_info.h @@ -57,7 +57,7 @@ register struct thread_info *__current_thread_info __asm__("$8"); #endif /* __ASSEMBLY__ */ -#define PREEMPT_ACTIVE 0x40000000 +#define PREEMPT_ACTIVE 0x10000000 /* * Thread information flags: diff --git a/include/asm-avr32/thread_info.h b/include/asm-avr32/thread_info.h index 07049f6c0d41..6ae2a825c0d0 100644 --- a/include/asm-avr32/thread_info.h +++ b/include/asm-avr32/thread_info.h @@ -70,7 +70,7 @@ static inline struct thread_info *current_thread_info(void) #endif /* !__ASSEMBLY__ */ -#define PREEMPT_ACTIVE 0x40000000 +#define PREEMPT_ACTIVE 0x10000000 /* * Thread information flags diff --git a/include/asm-generic/vmlinux.lds.h b/include/asm-generic/vmlinux.lds.h index 759f8f17df75..a683ecb6b25e 100644 --- a/include/asm-generic/vmlinux.lds.h +++ b/include/asm-generic/vmlinux.lds.h @@ -52,7 +52,10 @@ . = ALIGN(8); \ VMLINUX_SYMBOL(__start___markers) = .; \ *(__markers) \ - VMLINUX_SYMBOL(__stop___markers) = .; + VMLINUX_SYMBOL(__stop___markers) = .; \ + VMLINUX_SYMBOL(__start___imv) = .; \ + *(__imv) /* Immediate values: pointers */ \ + VMLINUX_SYMBOL(__stop___imv) = .; #define RO_DATA(align) \ . = ALIGN((align)); \ @@ -60,6 +63,10 @@ VMLINUX_SYMBOL(__start_rodata) = .; \ *(.rodata) *(.rodata.*) \ *(__vermagic) /* Kernel version magic */ \ + . = ALIGN(8); \ + VMLINUX_SYMBOL(__start___imv_cond_end) = .; \ + *(__imv_cond_end) /* Immediate condition end pointers */\ + VMLINUX_SYMBOL(__stop___imv_cond_end) = .; \ *(__markers_strings) /* Markers: strings */ \ } \ \ diff --git a/include/asm-powerpc/cacheflush.h b/include/asm-powerpc/cacheflush.h index ba667a383b8c..a52a565ce29a 100644 --- a/include/asm-powerpc/cacheflush.h +++ b/include/asm-powerpc/cacheflush.h @@ -63,7 +63,9 @@ extern void flush_dcache_phys_range(unsigned long start, unsigned long stop); #define copy_from_user_page(vma, page, vaddr, dst, src, len) \ memcpy(dst, src, len) - +#define text_poke memcpy +#define text_poke_early text_poke +#define sync_core() #ifdef CONFIG_DEBUG_PAGEALLOC /* internal debugging function */ diff --git a/include/asm-powerpc/immediate.h b/include/asm-powerpc/immediate.h new file mode 100644 index 000000000000..252206bc9c4e --- /dev/null +++ b/include/asm-powerpc/immediate.h @@ -0,0 +1,76 @@ +#ifndef _ASM_POWERPC_IMMEDIATE_H +#define _ASM_POWERPC_IMMEDIATE_H + +/* + * Immediate values. PowerPC architecture optimizations. + * + * (C) Copyright 2006 Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca> + * + * This file is released under the GPLv2. + * See the file COPYING for more details. + */ + +#include <asm/asm-compat.h> + +struct __imv { + unsigned long var; /* Identifier variable of the immediate value */ + unsigned long imv; /* + * Pointer to the memory location that holds + * the immediate value within the load immediate + * instruction. + */ + unsigned char size; /* Type size. */ +} __attribute__ ((packed)); + +/** + * imv_read - read immediate variable + * @name: immediate value name + * + * Reads the value of @name. + * Optimized version of the immediate. + * Do not use in __init and __exit functions. Use _imv_read() instead. + * Makes sure the 2 bytes update will be atomic by aligning the immediate + * value. Use a normal memory read for the 4 bytes immediate because there is no + * way to atomically update it without using a seqlock read side, which would + * cost more in term of total i-cache and d-cache space than a simple memory + * read. + */ +#define imv_read(name) \ + ({ \ + __typeof__(name##__imv) value; \ + BUILD_BUG_ON(sizeof(value) > 8); \ + switch (sizeof(value)) { \ + case 1: \ + asm(".section __imv,\"aw\",@progbits\n\t" \ + PPC_LONG "%c1, ((1f)-1)\n\t" \ + ".byte 1\n\t" \ + ".previous\n\t" \ + "li %0,0\n\t" \ + "1:\n\t" \ + : "=r" (value) \ + : "i" (&name##__imv)); \ + break; \ + case 2: \ + asm(".section __imv,\"aw\",@progbits\n\t" \ + PPC_LONG "%c1, ((1f)-2)\n\t" \ + ".byte 2\n\t" \ + ".previous\n\t" \ + ".align 2\n\t" \ + "li %0,0\n\t" \ + "1:\n\t" \ + : "=r" (value) \ + : "i" (&name##__imv)); \ + break; \ + case 4: \ + case 8: value = name##__imv; \ + break; \ + }; \ + value; \ + }) + +#define imv_cond(name) imv_read(name) +#define imv_cond_end() + +extern int arch_imv_update(const struct __imv *imv, int early); + +#endif /* _ASM_POWERPC_IMMEDIATE_H */ diff --git a/include/asm-x86/immediate.h b/include/asm-x86/immediate.h new file mode 100644 index 000000000000..1700a76089d2 --- /dev/null +++ b/include/asm-x86/immediate.h @@ -0,0 +1,169 @@ +#ifndef _ASM_X86_IMMEDIATE_H +#define _ASM_X86_IMMEDIATE_H + +/* + * Immediate values. x86 architecture optimizations. + * + * (C) Copyright 2006 Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca> + * + * This file is released under the GPLv2. + * See the file COPYING for more details. + */ + +#include <asm/asm.h> + +struct __imv { + unsigned long var; /* Pointer to the identifier variable of the + * immediate value + */ + unsigned long imv; /* + * Pointer to the memory location of the + * immediate value within the instruction. + */ + unsigned char var_size; /* Type size of variable. */ + unsigned char size; /* Type size of immediate value. */ + unsigned char insn_size;/* Instruction size. */ +} __attribute__ ((packed)); + +/** + * imv_read - read immediate variable + * @name: immediate value name + * + * Reads the value of @name. + * Optimized version of the immediate. + * Do not use in __init and __exit functions. Use _imv_read() instead. + * If size is bigger than the architecture long size, fall back on a memory + * read. + * + * Make sure to populate the initial static 64 bits opcode with a value + * what will generate an instruction with 8 bytes immediate value (not the REX.W + * prefixed one that loads a sign extended 32 bits immediate value in a r64 + * register). + * + * Create the instruction in a discarded section to calculate its size. This is + * how we can align the beginning of the instruction on an address that will + * permit atomic modification of the immediate value without knowing the size of + * the opcode used by the compiler. The operand size is known in advance. + */ +#define imv_read(name) \ + ({ \ + __typeof__(name##__imv) value; \ + BUILD_BUG_ON(sizeof(value) > 8); \ + switch (sizeof(value)) { \ + case 1: \ + asm(".section __discard,\"\",@progbits\n\t" \ + "1:\n\t" \ + "mov $0,%0\n\t" \ + "2:\n\t" \ + ".previous\n\t" \ + ".section __imv,\"aw\",@progbits\n\t" \ + _ASM_PTR "%c1, (3f)-%c2\n\t" \ + ".byte %c2, %c2, (2b-1b)\n\t" \ + ".previous\n\t" \ + "mov $0,%0\n\t" \ + "3:\n\t" \ + : "=q" (value) \ + : "i" (&name##__imv), \ + "i" (sizeof(value))); \ + break; \ + case 2: \ + case 4: \ + asm(".section __discard,\"\",@progbits\n\t" \ + "1:\n\t" \ + "mov $0,%0\n\t" \ + "2:\n\t" \ + ".previous\n\t" \ + ".section __imv,\"aw\",@progbits\n\t" \ + _ASM_PTR "%c1, (3f)-%c2\n\t" \ + ".byte %c2, %c2, (2b-1b)\n\t" \ + ".previous\n\t" \ + ".org . + ((-.-(2b-1b)) & (%c2-1)), 0x90\n\t" \ + "mov $0,%0\n\t" \ + "3:\n\t" \ + : "=r" (value) \ + : "i" (&name##__imv), \ + "i" (sizeof(value))); \ + break; \ + case 8: \ + if (sizeof(long) < 8) { \ + value = name##__imv; \ + break; \ + } \ + asm(".section __discard,\"\",@progbits\n\t" \ + "1:\n\t" \ + "mov $0xFEFEFEFE01010101,%0\n\t" \ + "2:\n\t" \ + ".previous\n\t" \ + ".section __imv,\"aw\",@progbits\n\t" \ + _ASM_PTR "%c1, (3f)-%c2\n\t" \ + ".byte %c2, %c2, (2b-1b)\n\t" \ + ".previous\n\t" \ + ".org . + ((-.-(2b-1b)) & (%c2-1)), 0x90\n\t" \ + "mov $0xFEFEFEFE01010101,%0\n\t" \ + "3:\n\t" \ + : "=r" (value) \ + : "i" (&name##__imv), \ + "i" (sizeof(value))); \ + break; \ + }; \ + value; \ + }) + +/* + * Uses %eax. + * immediate value size is declared as 0. + * Use in + * if (unlikely(imv_cond(var))) { + * imv_cond_end(); + * ... + * } else { + * imv_cond_end(); + * ... + * } + * Given a char as argument. + * If the expected code pattern insuring correct liveliness of ZF and %eax isn't + * met, fallback on standard immediate value. + * patches the 5 bytes mov for a e9 XX XX XX XX (near jump) + * Note : Patching the the 4 bytes immediate value with 1 byte variable + * on fallback. + */ +#define imv_cond(name) \ + ({ \ + uint32_t value; \ + BUILD_BUG_ON(sizeof(__typeof__(name##__imv)) > 1); \ + asm (".section __discard,\"\",@progbits\n\t" \ + "1:\n\t" \ + "mov $0,%0\n\t" \ + "2:\n\t" \ + ".previous\n\t" \ + ".section __imv,\"aw\",@progbits\n\t" \ + _ASM_PTR "%c1, (3f)-%c2\n\t" \ + ".byte %c3, 0, (2b-1b)\n\t" \ + ".previous\n\t" \ + "mov $0,%0\n\t" \ + "3:\n\t" \ + : "=a" (value) \ + : "i" (&name##__imv), \ + "i" (sizeof(value)), \ + "i" (sizeof(__typeof__(name##__imv)))); \ + value; \ + }) + +/* + * Make sure the %eax register and ZF are not live anymore at the current + * address, which is declared in the __imv_cond_end section. + * All asm statements clobbers the flags, but add "cc" clobber just to be sure. + * Clobbers %eax. + */ +#define imv_cond_end() \ + do { \ + asm (".section __imv_cond_end,\"a\",@progbits\n\t" \ + _ASM_PTR "1f\n\t" \ + ".previous\n\t" \ + "1:\n\t" \ + : : : "eax", "cc"); \ + } while (0) + +extern int arch_imv_update(struct __imv *imv, int early); + +#endif /* _ASM_X86_IMMEDIATE_H */ diff --git a/include/asm-x86/ipi.h b/include/asm-x86/ipi.h index ecc80f341f37..5f7310aa3efd 100644 --- a/include/asm-x86/ipi.h +++ b/include/asm-x86/ipi.h @@ -121,7 +121,7 @@ static inline void send_IPI_mask_sequence(cpumask_t mask, int vector) * - mbligh */ local_irq_save(flags); - for_each_cpu_mask(query_cpu, mask) { + for_each_cpu_mask_nr(query_cpu, mask) { __send_IPI_dest_field(per_cpu(x86_cpu_to_apicid, query_cpu), vector, APIC_DEST_PHYSICAL); } diff --git a/include/asm-x86/irqflags.h b/include/asm-x86/irqflags.h index 24d71b1eb189..c3009fd85a47 100644 --- a/include/asm-x86/irqflags.h +++ b/include/asm-x86/irqflags.h @@ -51,6 +51,61 @@ static inline void native_halt(void) #endif +#ifdef CONFIG_X86_64 +/* + * Only returns from a trap or exception to a NMI context (intra-privilege + * level near return) to the same SS and CS segments. Should be used + * upon trap or exception return when nested over a NMI context so no iret is + * issued. It takes care of modifying the eflags, rsp and returning to the + * previous function. + * + * The stack, at that point, looks like : + * + * 0(rsp) RIP + * 8(rsp) CS + * 16(rsp) EFLAGS + * 24(rsp) RSP + * 32(rsp) SS + * + * Upon execution : + * Copy EIP to the top of the return stack + * Update top of return stack address + * Pop eflags into the eflags register + * Make the return stack current + * Near return (popping the return address from the return stack) + */ +#define NATIVE_INTERRUPT_RETURN_NMI_SAFE pushq %rax; \ + movq %rsp, %rax; \ + movq 24+8(%rax), %rsp; \ + pushq 0+8(%rax); \ + pushq 16+8(%rax); \ + movq (%rax), %rax; \ + popfq; \ + ret +#else +/* + * Protected mode only, no V8086. Implies that protected mode must + * be entered before NMIs or MCEs are enabled. Only returns from a trap or + * exception to a NMI context (intra-privilege level far return). Should be used + * upon trap or exception return when nested over a NMI context so no iret is + * issued. + * + * The stack, at that point, looks like : + * + * 0(esp) EIP + * 4(esp) CS + * 8(esp) EFLAGS + * + * Upon execution : + * Copy the stack eflags to top of stack + * Pop eflags into the eflags register + * Far return: pop EIP and CS into their register, and additionally pop EFLAGS. + */ +#define NATIVE_INTERRUPT_RETURN_NMI_SAFE pushl 8(%esp); \ + popfl; \ + lret $4 +#endif + #ifdef CONFIG_PARAVIRT #include <asm/paravirt.h> #else @@ -109,6 +164,7 @@ static inline unsigned long __raw_local_irq_save(void) #define ENABLE_INTERRUPTS(x) sti #define DISABLE_INTERRUPTS(x) cli +#define INTERRUPT_RETURN_NMI_SAFE NATIVE_INTERRUPT_RETURN_NMI_SAFE #ifdef CONFIG_X86_64 #define INTERRUPT_RETURN iretq diff --git a/include/asm-x86/kdebug.h b/include/asm-x86/kdebug.h index fe1fbdec1e1c..fa382ed7e2b2 100644 --- a/include/asm-x86/kdebug.h +++ b/include/asm-x86/kdebug.h @@ -3,6 +3,9 @@ #include <linux/notifier.h> +#include <linux/ptrace.h> +#include <asm/system.h> + struct pt_regs; /* Grossly misnamed. */ @@ -34,4 +37,13 @@ extern void show_regs(struct pt_regs *regs); extern unsigned long oops_begin(void); extern void oops_end(unsigned long, struct pt_regs *, int signr); +/* trap3/1 are intr gates for kprobes. So, restore the status of IF, + * if necessary, before executing the original int3/1 (trap) handler. + */ +static inline void restore_interrupts(struct pt_regs *regs) +{ + if (regs->flags & X86_EFLAGS_IF) + local_irq_enable(); +} + #endif diff --git a/include/asm-x86/kprobes.h b/include/asm-x86/kprobes.h index 54980b0b3892..93a7f49d3e25 100644 --- a/include/asm-x86/kprobes.h +++ b/include/asm-x86/kprobes.h @@ -82,15 +82,6 @@ struct kprobe_ctlblk { struct prev_kprobe prev_kprobe; }; -/* trap3/1 are intr gates for kprobes. So, restore the status of IF, - * if necessary, before executing the original int3/1 (trap) handler. - */ -static inline void restore_interrupts(struct pt_regs *regs) -{ - if (regs->flags & X86_EFLAGS_IF) - local_irq_enable(); -} - extern int kprobe_fault_handler(struct pt_regs *regs, int trapnr); extern int kprobe_exceptions_notify(struct notifier_block *self, unsigned long val, void *data); diff --git a/include/asm-x86/numa_64.h b/include/asm-x86/numa_64.h index 22e87c9f6a80..b510daf4f4d8 100644 --- a/include/asm-x86/numa_64.h +++ b/include/asm-x86/numa_64.h @@ -14,11 +14,9 @@ extern int compute_hash_shift(struct bootnode *nodes, int numblks, #define ZONE_ALIGN (1UL << (MAX_ORDER+PAGE_SHIFT)) -extern void numa_add_cpu(int cpu); extern void numa_init_array(void); extern int numa_off; -extern void numa_set_node(int cpu, int node); extern void srat_reserve_add_area(int nodeid); extern int hotadd_percent; @@ -31,15 +29,16 @@ extern void setup_node_bootmem(int nodeid, unsigned long start, #ifdef CONFIG_NUMA extern void __init init_cpu_to_node(void); - -static inline void clear_node_cpumask(int cpu) -{ - clear_bit(cpu, (unsigned long *)&node_to_cpumask_map[cpu_to_node(cpu)]); -} - +extern void __cpuinit numa_set_node(int cpu, int node); +extern void __cpuinit numa_clear_node(int cpu); +extern void __cpuinit numa_add_cpu(int cpu); +extern void __cpuinit numa_remove_cpu(int cpu); #else -#define init_cpu_to_node() do {} while (0) -#define clear_node_cpumask(cpu) do {} while (0) +static inline void init_cpu_to_node(void) { } +static inline void numa_set_node(int cpu, int node) { } +static inline void numa_clear_node(int cpu) { } +static inline void numa_add_cpu(int cpu, int node) { } +static inline void numa_remove_cpu(int cpu) { } #endif #endif diff --git a/include/asm-x86/paravirt.h b/include/asm-x86/paravirt.h index 0f13b945e240..d5087e041117 100644 --- a/include/asm-x86/paravirt.h +++ b/include/asm-x86/paravirt.h @@ -141,9 +141,10 @@ struct pv_cpu_ops { u64 (*read_pmc)(int counter); unsigned long long (*read_tscp)(unsigned int *aux); - /* These two are jmp to, not actually called. */ + /* These three are jmp to, not actually called. */ void (*irq_enable_syscall_ret)(void); void (*iret)(void); + void (*nmi_return)(void); void (*swapgs)(void); @@ -1385,6 +1386,10 @@ static inline unsigned long __raw_local_irq_save(void) PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_iret), CLBR_NONE, \ jmp *%cs:pv_cpu_ops+PV_CPU_iret) +#define INTERRUPT_RETURN_NMI_SAFE \ + PARA_SITE(PARA_PATCH(pv_cpu_ops, PV_CPU_nmi_return), CLBR_NONE, \ + jmp *%cs:pv_cpu_ops+PV_CPU_nmi_return) + #define DISABLE_INTERRUPTS(clobbers) \ PARA_SITE(PARA_PATCH(pv_irq_ops, PV_IRQ_irq_disable), clobbers, \ PV_SAVE_REGS; \ diff --git a/include/asm-x86/pda.h b/include/asm-x86/pda.h index 62b734986a44..385ffbe4c169 100644 --- a/include/asm-x86/pda.h +++ b/include/asm-x86/pda.h @@ -20,6 +20,8 @@ struct x8664_pda { /* gcc-ABI: this canary MUST be at offset 40!!! */ char *irqstackptr; + short nodenumber; /* number of current node (32k max) */ + short in_bootmem; /* pda lives in bootmem */ unsigned int __softirq_pending; unsigned int __nmi_count; /* number of NMI on this CPUs */ short mmu_state; @@ -35,8 +37,7 @@ struct x8664_pda { unsigned irq_spurious_count; } ____cacheline_aligned_in_smp; -extern struct x8664_pda *_cpu_pda[]; -extern struct x8664_pda boot_cpu_pda[]; +extern struct x8664_pda **_cpu_pda; extern void pda_init(int); #define cpu_pda(i) (_cpu_pda[i]) diff --git a/include/asm-x86/percpu.h b/include/asm-x86/percpu.h index 736fc3bb8e1e..912a3a17b9db 100644 --- a/include/asm-x86/percpu.h +++ b/include/asm-x86/percpu.h @@ -143,4 +143,50 @@ do { \ #define x86_or_percpu(var, val) percpu_to_op("or", per_cpu__##var, val) #endif /* !__ASSEMBLY__ */ #endif /* !CONFIG_X86_64 */ + +#ifdef CONFIG_SMP + +/* + * Define the "EARLY_PER_CPU" macros. These are used for some per_cpu + * variables that are initialized and accessed before there are per_cpu + * areas allocated. + */ + +#define DEFINE_EARLY_PER_CPU(_type, _name, _initvalue) \ + DEFINE_PER_CPU(_type, _name) = _initvalue; \ + __typeof__(_type) _name##_early_map[NR_CPUS] __initdata = \ + { [0 ... NR_CPUS-1] = _initvalue }; \ + __typeof__(_type) *_name##_early_ptr = _name##_early_map + +#define EXPORT_EARLY_PER_CPU_SYMBOL(_name) \ + EXPORT_PER_CPU_SYMBOL(_name) + +#define DECLARE_EARLY_PER_CPU(_type, _name) \ + DECLARE_PER_CPU(_type, _name); \ + extern __typeof__(_type) *_name##_early_ptr; \ + extern __typeof__(_type) _name##_early_map[] + +#define early_per_cpu_ptr(_name) (_name##_early_ptr) +#define early_per_cpu_map(_name, _idx) (_name##_early_map[_idx]) +#define early_per_cpu(_name, _cpu) \ + (early_per_cpu_ptr(_name) ? \ + early_per_cpu_ptr(_name)[_cpu] : \ + per_cpu(_name, _cpu)) + +#else /* !CONFIG_SMP */ +#define DEFINE_EARLY_PER_CPU(_type, _name, _initvalue) \ + DEFINE_PER_CPU(_type, _name) = _initvalue + +#define EXPORT_EARLY_PER_CPU_SYMBOL(_name) \ + EXPORT_PER_CPU_SYMBOL(_name) + +#define DECLARE_EARLY_PER_CPU(_type, _name) \ + DECLARE_PER_CPU(_type, _name) + +#define early_per_cpu(_name, _cpu) per_cpu(_name, _cpu) +#define early_per_cpu_ptr(_name) NULL +/* no early_per_cpu_map() */ + +#endif /* !CONFIG_SMP */ + #endif /* _ASM_X86_PERCPU_H_ */ diff --git a/include/asm-x86/smp.h b/include/asm-x86/smp.h index 1ebaa5cd3112..ec841639fb44 100644 --- a/include/asm-x86/smp.h +++ b/include/asm-x86/smp.h @@ -29,21 +29,12 @@ extern int smp_num_siblings; extern unsigned int num_processors; extern cpumask_t cpu_initialized; -#ifdef CONFIG_SMP -extern u16 x86_cpu_to_apicid_init[]; -extern u16 x86_bios_cpu_apicid_init[]; -extern void *x86_cpu_to_apicid_early_ptr; -extern void *x86_bios_cpu_apicid_early_ptr; -#else -#define x86_cpu_to_apicid_early_ptr NULL -#define x86_bios_cpu_apicid_early_ptr NULL -#endif - DECLARE_PER_CPU(cpumask_t, cpu_sibling_map); DECLARE_PER_CPU(cpumask_t, cpu_core_map); DECLARE_PER_CPU(u16, cpu_llc_id); -DECLARE_PER_CPU(u16, x86_cpu_to_apicid); -DECLARE_PER_CPU(u16, x86_bios_cpu_apicid); + +DECLARE_EARLY_PER_CPU(u16, x86_cpu_to_apicid); +DECLARE_EARLY_PER_CPU(u16, x86_bios_cpu_apicid); /* Static state in head.S used to set up a CPU */ extern struct { diff --git a/include/asm-x86/topology.h b/include/asm-x86/topology.h index dcf3f8131d6b..1f97758de4ab 100644 --- a/include/asm-x86/topology.h +++ b/include/asm-x86/topology.h @@ -35,79 +35,88 @@ # endif #endif +/* Node not present */ +#define NUMA_NO_NODE (-1) + #ifdef CONFIG_NUMA #include <linux/cpumask.h> #include <asm/mpspec.h> -/* Mappings between logical cpu number and node number */ #ifdef CONFIG_X86_32 -extern int cpu_to_node_map[]; -#else -/* Returns the number of the current Node. */ -#define numa_node_id() (early_cpu_to_node(raw_smp_processor_id())) -#endif - -DECLARE_PER_CPU(int, x86_cpu_to_node_map); - -#ifdef CONFIG_SMP -extern int x86_cpu_to_node_map_init[]; -extern void *x86_cpu_to_node_map_early_ptr; -#else -#define x86_cpu_to_node_map_early_ptr NULL -#endif +/* Mappings between node number and cpus on that node. */ extern cpumask_t node_to_cpumask_map[]; -#define NUMA_NO_NODE (-1) +/* Mappings between logical cpu number and node number */ +extern int cpu_to_node_map[]; /* Returns the number of the node containing CPU 'cpu' */ -#ifdef CONFIG_X86_32 -#define early_cpu_to_node(cpu) cpu_to_node(cpu) static inline int cpu_to_node(int cpu) { return cpu_to_node_map[cpu]; } +#define early_cpu_to_node(cpu) cpu_to_node(cpu) -#else /* CONFIG_X86_64 */ - -#ifdef CONFIG_SMP -static inline int early_cpu_to_node(int cpu) +/* Returns a bitmask of CPUs on Node 'node'. */ +static inline cpumask_t node_to_cpumask(int node) { - int *cpu_to_node_map = x86_cpu_to_node_map_early_ptr; - - if (cpu_to_node_map) - return cpu_to_node_map[cpu]; - else if (per_cpu_offset(cpu)) - return per_cpu(x86_cpu_to_node_map, cpu); - else - return NUMA_NO_NODE; + return node_to_cpumask_map[node]; } -#else -#define early_cpu_to_node(cpu) cpu_to_node(cpu) -#endif +#else /* CONFIG_X86_64 */ + +/* Mappings between node number and cpus on that node. */ +extern cpumask_t *node_to_cpumask_map; + +/* Mappings between logical cpu number and node number */ +DECLARE_EARLY_PER_CPU(int, x86_cpu_to_node_map); + +/* Returns the number of the current Node. */ +#define numa_node_id() read_pda(nodenumber) + +#ifdef CONFIG_DEBUG_PER_CPU_MAPS +extern int cpu_to_node(int cpu); +extern int early_cpu_to_node(int cpu); +extern cpumask_t *_node_to_cpumask_ptr(int node); +extern cpumask_t node_to_cpumask(int node); + +#else /* !CONFIG_DEBUG_PER_CPU_MAPS */ + +/* Returns the number of the node containing CPU 'cpu' */ static inline int cpu_to_node(int cpu) { -#ifdef CONFIG_DEBUG_PER_CPU_MAPS - if (x86_cpu_to_node_map_early_ptr) { - printk("KERN_NOTICE cpu_to_node(%d): usage too early!\n", - (int)cpu); - dump_stack(); - return ((int *)x86_cpu_to_node_map_early_ptr)[cpu]; - } -#endif return per_cpu(x86_cpu_to_node_map, cpu); } -#ifdef CONFIG_NUMA +/* Same function but used if called before per_cpu areas are setup */ +static inline int early_cpu_to_node(int cpu) +{ + if (early_per_cpu_ptr(x86_cpu_to_node_map)) + return early_per_cpu_ptr(x86_cpu_to_node_map)[cpu]; + + return per_cpu(x86_cpu_to_node_map, cpu); +} /* Returns a pointer to the cpumask of CPUs on Node 'node'. */ +static inline cpumask_t *_node_to_cpumask_ptr(int node) +{ + return &node_to_cpumask_map[node]; +} + +/* Returns a bitmask of CPUs on Node 'node'. */ +static inline cpumask_t node_to_cpumask(int node) +{ + return node_to_cpumask_map[node]; +} + +#endif /* !CONFIG_DEBUG_PER_CPU_MAPS */ + +/* Replace default node_to_cpumask_ptr with optimized version */ #define node_to_cpumask_ptr(v, node) \ - cpumask_t *v = &(node_to_cpumask_map[node]) + cpumask_t *v = _node_to_cpumask_ptr(node) #define node_to_cpumask_ptr_next(v, node) \ - v = &(node_to_cpumask_map[node]) -#endif + v = _node_to_cpumask_ptr(node) #endif /* CONFIG_X86_64 */ @@ -117,20 +126,6 @@ static inline int cpu_to_node(int cpu) */ #define parent_node(node) (node) -/* Returns a bitmask of CPUs on Node 'node'. */ -static inline cpumask_t node_to_cpumask(int node) -{ - return node_to_cpumask_map[node]; -} - -/* Returns the number of the first CPU on Node 'node'. */ -static inline int node_to_first_cpu(int node) -{ - cpumask_t mask = node_to_cpumask(node); - - return first_cpu(mask); -} - #define pcibus_to_node(bus) __pcibus_to_node(bus) #define pcibus_to_cpumask(bus) __pcibus_to_cpumask(bus) @@ -180,12 +175,44 @@ extern int __node_distance(int, int); #define node_distance(a, b) __node_distance(a, b) #endif -#else /* CONFIG_NUMA */ +#else /* !CONFIG_NUMA */ +#define numa_node_id() 0 +#define cpu_to_node(cpu) 0 +#define early_cpu_to_node(cpu) 0 + +static inline cpumask_t *_node_to_cpumask_ptr(int node) +{ + return &cpu_online_map; +} +static inline cpumask_t node_to_cpumask(int node) +{ + return cpu_online_map; +} +static inline int node_to_first_cpu(int node) +{ + return first_cpu(cpu_online_map); +} + +/* Replace default node_to_cpumask_ptr with optimized version */ +#define node_to_cpumask_ptr(v, node) \ + cpumask_t *v = _node_to_cpumask_ptr(node) + +#define node_to_cpumask_ptr_next(v, node) \ + v = _node_to_cpumask_ptr(node) #endif #include <asm-generic/topology.h> +#ifdef CONFIG_NUMA +/* Returns the number of the first CPU on Node 'node'. */ +static inline int node_to_first_cpu(int node) +{ + node_to_cpumask_ptr(mask, node); + return first_cpu(*mask); +} +#endif + extern cpumask_t cpu_coregroup_map(int cpu); #ifdef ENABLE_TOPO_DEFINES @@ -193,6 +220,9 @@ extern cpumask_t cpu_coregroup_map(int cpu); #define topology_core_id(cpu) (cpu_data(cpu).cpu_core_id) #define topology_core_siblings(cpu) (per_cpu(cpu_core_map, cpu)) #define topology_thread_siblings(cpu) (per_cpu(cpu_sibling_map, cpu)) + +/* indicates that pointers to the topology cpumask_t maps are valid */ +#define arch_provides_topology_pointers yes #endif static inline void arch_fix_phys_package_id(int num, u32 slot) @@ -220,4 +250,4 @@ static inline void set_mp_bus_to_node(int busnum, int node) } #endif -#endif +#endif /* _ASM_X86_TOPOLOGY_H */ diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h index 9650806fe2ea..2174d39e39d1 100644 --- a/include/linux/cpumask.h +++ b/include/linux/cpumask.h @@ -17,6 +17,20 @@ * For details of cpus_onto(), see bitmap_onto in lib/bitmap.c. * For details of cpus_fold(), see bitmap_fold in lib/bitmap.c. * + * . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + * Note: The alternate operations with the suffix "_nr" are used + * to limit the range of the loop to nr_cpu_ids instead of + * NR_CPUS when NR_CPUS > 64 for performance reasons. + * If NR_CPUS is <= 64 then most assembler bitmask + * operators execute faster with a constant range, so + * the operator will continue to use NR_CPUS. + * + * Another consideration is that nr_cpu_ids is initialized + * to NR_CPUS and isn't lowered until the possible cpus are + * discovered (including any disabled cpus). So early uses + * will span the entire range of NR_CPUS. + * . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . + * * The available cpumask operations are: * * void cpu_set(cpu, mask) turn on bit 'cpu' in mask @@ -38,12 +52,14 @@ * int cpus_empty(mask) Is mask empty (no bits sets)? * int cpus_full(mask) Is mask full (all bits sets)? * int cpus_weight(mask) Hamming weigh - number of set bits + * int cpus_weight_nr(mask) Same using nr_cpu_ids instead of NR_CPUS * * void cpus_shift_right(dst, src, n) Shift right * void cpus_shift_left(dst, src, n) Shift left * * int first_cpu(mask) Number lowest set bit, or NR_CPUS * int next_cpu(cpu, mask) Next cpu past 'cpu', or NR_CPUS + * int next_cpu_nr(cpu, mask) Next cpu past 'cpu', or nr_cpu_ids * * cpumask_t cpumask_of_cpu(cpu) Return cpumask with bit 'cpu' set * CPU_MASK_ALL Initializer - all bits set @@ -59,7 +75,8 @@ * void cpus_onto(dst, orig, relmap) *dst = orig relative to relmap * void cpus_fold(dst, orig, sz) dst bits = orig bits mod sz * - * for_each_cpu_mask(cpu, mask) for-loop cpu over mask + * for_each_cpu_mask(cpu, mask) for-loop cpu over mask using NR_CPUS + * for_each_cpu_mask_nr(cpu, mask) for-loop cpu over mask using nr_cpu_ids * * int num_online_cpus() Number of online CPUs * int num_possible_cpus() Number of all possible CPUs @@ -216,15 +233,6 @@ static inline void __cpus_shift_left(cpumask_t *dstp, bitmap_shift_left(dstp->bits, srcp->bits, n, nbits); } -#ifdef CONFIG_SMP -int __first_cpu(const cpumask_t *srcp); -#define first_cpu(src) __first_cpu(&(src)) -int __next_cpu(int n, const cpumask_t *srcp); -#define next_cpu(n, src) __next_cpu((n), &(src)) -#else -#define first_cpu(src) ({ (void)(src); 0; }) -#define next_cpu(n, src) ({ (void)(src); 1; }) -#endif #ifdef CONFIG_HAVE_CPUMASK_OF_CPU_MAP extern cpumask_t *cpumask_of_cpu_map; @@ -350,15 +358,48 @@ static inline void __cpus_fold(cpumask_t *dstp, const cpumask_t *origp, bitmap_fold(dstp->bits, origp->bits, sz, nbits); } -#if NR_CPUS > 1 +#if NR_CPUS == 1 + +#define nr_cpu_ids 1 +#define first_cpu(src) ({ (void)(src); 0; }) +#define next_cpu(n, src) ({ (void)(src); 1; }) +#define any_online_cpu(mask) 0 +#define for_each_cpu_mask(cpu, mask) \ + for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask) + +#else /* NR_CPUS > 1 */ + +extern int nr_cpu_ids; +int __first_cpu(const cpumask_t *srcp); +int __next_cpu(int n, const cpumask_t *srcp); +int __any_online_cpu(const cpumask_t *mask); + +#define first_cpu(src) __first_cpu(&(src)) +#define next_cpu(n, src) __next_cpu((n), &(src)) +#define any_online_cpu(mask) __any_online_cpu(&(mask)) #define for_each_cpu_mask(cpu, mask) \ for ((cpu) = first_cpu(mask); \ (cpu) < NR_CPUS; \ (cpu) = next_cpu((cpu), (mask))) -#else /* NR_CPUS == 1 */ -#define for_each_cpu_mask(cpu, mask) \ - for ((cpu) = 0; (cpu) < 1; (cpu)++, (void)mask) -#endif /* NR_CPUS */ +#endif + +#if NR_CPUS <= 64 + +#define next_cpu_nr(n, src) next_cpu(n, src) +#define cpus_weight_nr(cpumask) cpus_weight(cpumask) +#define for_each_cpu_mask_nr(cpu, mask) for_each_cpu_mask(cpu, mask) + +#else /* NR_CPUS > 64 */ + +int __next_cpu_nr(int n, const cpumask_t *srcp); +#define next_cpu_nr(n, src) __next_cpu_nr((n), &(src)) +#define cpus_weight_nr(cpumask) __cpus_weight(&(cpumask), nr_cpu_ids) +#define for_each_cpu_mask_nr(cpu, mask) \ + for ((cpu) = first_cpu(mask); \ + (cpu) < nr_cpu_ids; \ + (cpu) = next_cpu_nr((cpu), (mask))) + +#endif /* NR_CPUS > 64 */ /* * The following particular system cpumasks and operations manage @@ -419,14 +460,16 @@ static inline void __cpus_fold(cpumask_t *dstp, const cpumask_t *origp, extern cpumask_t cpu_possible_map; extern cpumask_t cpu_online_map; extern cpumask_t cpu_present_map; +extern cpumask_t cpu_system_map; #if NR_CPUS > 1 -#define num_online_cpus() cpus_weight(cpu_online_map) -#define num_possible_cpus() cpus_weight(cpu_possible_map) -#define num_present_cpus() cpus_weight(cpu_present_map) +#define num_online_cpus() cpus_weight_nr(cpu_online_map) +#define num_possible_cpus() cpus_weight_nr(cpu_possible_map) +#define num_present_cpus() cpus_weight_nr(cpu_present_map) #define cpu_online(cpu) cpu_isset((cpu), cpu_online_map) #define cpu_possible(cpu) cpu_isset((cpu), cpu_possible_map) #define cpu_present(cpu) cpu_isset((cpu), cpu_present_map) +#define cpu_system(cpu) cpu_isset((cpu), cpu_system_map) #else #define num_online_cpus() 1 #define num_possible_cpus() 1 @@ -434,21 +477,15 @@ extern cpumask_t cpu_present_map; #define cpu_online(cpu) ((cpu) == 0) #define cpu_possible(cpu) ((cpu) == 0) #define cpu_present(cpu) ((cpu) == 0) +#define cpu_system(cpu) ((cpu) == 0) #endif -#define cpu_is_offline(cpu) unlikely(!cpu_online(cpu)) +extern int cpus_match_system(cpumask_t mask); -#ifdef CONFIG_SMP -extern int nr_cpu_ids; -#define any_online_cpu(mask) __any_online_cpu(&(mask)) -int __any_online_cpu(const cpumask_t *mask); -#else -#define nr_cpu_ids 1 -#define any_online_cpu(mask) 0 -#endif +#define cpu_is_offline(cpu) unlikely(!cpu_online(cpu)) -#define for_each_possible_cpu(cpu) for_each_cpu_mask((cpu), cpu_possible_map) -#define for_each_online_cpu(cpu) for_each_cpu_mask((cpu), cpu_online_map) -#define for_each_present_cpu(cpu) for_each_cpu_mask((cpu), cpu_present_map) +#define for_each_possible_cpu(cpu) for_each_cpu_mask_nr((cpu), cpu_possible_map) +#define for_each_online_cpu(cpu) for_each_cpu_mask_nr((cpu), cpu_online_map) +#define for_each_present_cpu(cpu) for_each_cpu_mask_nr((cpu), cpu_present_map) #endif /* __LINUX_CPUMASK_H */ diff --git a/include/linux/cpuset.h b/include/linux/cpuset.h index 038578362b47..712f25c16b8d 100644 --- a/include/linux/cpuset.h +++ b/include/linux/cpuset.h @@ -78,6 +78,8 @@ extern void cpuset_track_online_nodes(void); extern int current_cpuset_is_being_rebound(void); +extern struct blocking_notifier_head system_map_notifier; + #else /* !CONFIG_CPUSETS */ static inline int cpuset_init_early(void) { return 0; } diff --git a/include/linux/dcache.h b/include/linux/dcache.h index 2a6639407c80..1f5cebf10a23 100644 --- a/include/linux/dcache.h +++ b/include/linux/dcache.h @@ -3,6 +3,7 @@ #include <asm/atomic.h> #include <linux/list.h> +#include <linux/rculist.h> #include <linux/spinlock.h> #include <linux/cache.h> #include <linux/rcupdate.h> diff --git a/include/linux/hardirq.h b/include/linux/hardirq.h index 897f723bd222..e15b0ea6073b 100644 --- a/include/linux/hardirq.h +++ b/include/linux/hardirq.h @@ -22,10 +22,13 @@ * PREEMPT_MASK: 0x000000ff * SOFTIRQ_MASK: 0x0000ff00 * HARDIRQ_MASK: 0x0fff0000 + * HARDNMI_MASK: 0x40000000 */ #define PREEMPT_BITS 8 #define SOFTIRQ_BITS 8 +#define HARDNMI_BITS 1 + #ifndef HARDIRQ_BITS #define HARDIRQ_BITS 12 @@ -45,16 +48,19 @@ #define PREEMPT_SHIFT 0 #define SOFTIRQ_SHIFT (PREEMPT_SHIFT + PREEMPT_BITS) #define HARDIRQ_SHIFT (SOFTIRQ_SHIFT + SOFTIRQ_BITS) +#define HARDNMI_SHIFT (30) #define __IRQ_MASK(x) ((1UL << (x))-1) #define PREEMPT_MASK (__IRQ_MASK(PREEMPT_BITS) << PREEMPT_SHIFT) #define SOFTIRQ_MASK (__IRQ_MASK(SOFTIRQ_BITS) << SOFTIRQ_SHIFT) #define HARDIRQ_MASK (__IRQ_MASK(HARDIRQ_BITS) << HARDIRQ_SHIFT) +#define HARDNMI_MASK (__IRQ_MASK(HARDNMI_BITS) << HARDNMI_SHIFT) #define PREEMPT_OFFSET (1UL << PREEMPT_SHIFT) #define SOFTIRQ_OFFSET (1UL << SOFTIRQ_SHIFT) #define HARDIRQ_OFFSET (1UL << HARDIRQ_SHIFT) +#define HARDNMI_OFFSET (1UL << HARDNMI_SHIFT) #if PREEMPT_ACTIVE < (1 << (HARDIRQ_SHIFT + HARDIRQ_BITS)) #error PREEMPT_ACTIVE is too low! @@ -62,7 +68,9 @@ #define hardirq_count() (preempt_count() & HARDIRQ_MASK) #define softirq_count() (preempt_count() & SOFTIRQ_MASK) -#define irq_count() (preempt_count() & (HARDIRQ_MASK | SOFTIRQ_MASK)) +#define irq_count() \ + (preempt_count() & (HARDNMI_MASK | HARDIRQ_MASK | SOFTIRQ_MASK)) +#define hardnmi_count() (preempt_count() & HARDNMI_MASK) /* * Are we doing bottom half or hardware interrupt processing? @@ -71,6 +79,7 @@ #define in_irq() (hardirq_count()) #define in_softirq() (softirq_count()) #define in_interrupt() (irq_count()) +#define in_nmi() (hardnmi_count()) /* * Are we running in atomic context? WARNING: this macro cannot @@ -159,7 +168,19 @@ extern void irq_enter(void); */ extern void irq_exit(void); -#define nmi_enter() do { lockdep_off(); __irq_enter(); } while (0) -#define nmi_exit() do { __irq_exit(); lockdep_on(); } while (0) +#define nmi_enter() \ + do { \ + lockdep_off(); \ + BUG_ON(hardnmi_count()); \ + add_preempt_count(HARDNMI_OFFSET); \ + __irq_enter(); \ + } while (0) + +#define nmi_exit() \ + do { \ + __irq_exit(); \ + sub_preempt_count(HARDNMI_OFFSET); \ + lockdep_on(); \ + } while (0) #endif /* LINUX_HARDIRQ_H */ diff --git a/include/linux/immediate.h b/include/linux/immediate.h new file mode 100644 index 000000000000..3754e846eaa0 --- /dev/null +++ b/include/linux/immediate.h @@ -0,0 +1,97 @@ +#ifndef _LINUX_IMMEDIATE_H +#define _LINUX_IMMEDIATE_H + +/* + * Immediate values, can be updated at runtime and save cache lines. + * + * (C) Copyright 2007 Mathieu Desnoyers <mathieu.desnoyers@polymtl.ca> + * + * This file is released under the GPLv2. + * See the file COPYING for more details. + */ + +#ifdef CONFIG_IMMEDIATE + +#include <asm/immediate.h> + +/** + * imv_set - set immediate variable (with locking) + * @name: immediate value name + * @i: required value + * + * Sets the value of @name, taking the module_mutex if required by + * the architecture. + */ +#define imv_set(name, i) \ + do { \ + name##__imv = (i); \ + core_imv_update(); \ + module_imv_update(); \ + } while (0) + +/* + * Internal update functions. + */ +extern void core_imv_update(void); +extern void imv_update_range(struct __imv *begin, struct __imv *end); +extern void imv_unref_core_init(void); +extern void imv_unref(struct __imv *begin, struct __imv *end, void *start, + unsigned long size); +extern int _is_imv_cond_end(unsigned long *begin, unsigned long *end, + unsigned long addr1, unsigned long addr2); +extern int is_imv_cond_end(unsigned long addr1, unsigned long addr2); + +#else + +/* + * Generic immediate values: a simple, standard, memory load. + */ + +/** + * imv_read - read immediate variable + * @name: immediate value name + * + * Reads the value of @name. + */ +#define imv_read(name) _imv_read(name) + +/** + * imv_cond - read immediate variable use as condition for if() + * @name: immediate value name + * + * Reads the value of @name. + */ +#define imv_cond(name) _imv_read(name) +#define imv_cond_end() + +/** + * imv_set - set immediate variable (with locking) + * @name: immediate value name + * @i: required value + * + * Sets the value of @name, taking the module_mutex if required by + * the architecture. + */ +#define imv_set(name, i) (name##__imv = (i)) + +static inline void core_imv_update(void) { } +static inline void imv_unref_core_init(void) { } + +#endif + +#define DECLARE_IMV(type, name) extern __typeof__(type) name##__imv +#define DEFINE_IMV(type, name) __typeof__(type) name##__imv + +#define EXPORT_IMV_SYMBOL(name) EXPORT_SYMBOL(name##__imv) +#define EXPORT_IMV_SYMBOL_GPL(name) EXPORT_SYMBOL_GPL(name##__imv) + +/** + * _imv_read - Read immediate value with standard memory load. + * @name: immediate value name + * + * Force a data read of the immediate value instead of the immediate value + * based mechanism. Useful for __init and __exit section data read. + */ +#define _imv_read(name) (name##__imv) + +#endif diff --git a/include/linux/irq.h b/include/linux/irq.h index 552e0ec269c9..f8d5dc08c53a 100644 --- a/include/linux/irq.h +++ b/include/linux/irq.h @@ -244,14 +244,7 @@ static inline void set_balance_irq_affinity(unsigned int irq, cpumask_t mask) } #endif -#ifdef CONFIG_AUTO_IRQ_AFFINITY extern int select_smp_affinity(unsigned int irq); -#else -static inline int select_smp_affinity(unsigned int irq) -{ - return 1; -} -#endif extern int no_irq_affinity; diff --git a/include/linux/kernel.h b/include/linux/kernel.h index 7cb7ea58232f..a1ecdb8cc4f8 100644 --- a/include/linux/kernel.h +++ b/include/linux/kernel.h @@ -184,9 +184,6 @@ asmlinkage int vprintk(const char *fmt, va_list args) __attribute__ ((format (printf, 1, 0))); asmlinkage int printk(const char * fmt, ...) __attribute__ ((format (printf, 1, 2))) __cold; -extern int log_buf_get_len(void); -extern int log_buf_read(int idx); -extern int log_buf_copy(char *dest, int idx, int len); extern int printk_ratelimit_jiffies; extern int printk_ratelimit_burst; @@ -202,9 +199,6 @@ static inline int vprintk(const char *s, va_list args) { return 0; } static inline int printk(const char *s, ...) __attribute__ ((format (printf, 1, 2))); static inline int __cold printk(const char *s, ...) { return 0; } -static inline int log_buf_get_len(void) { return 0; } -static inline int log_buf_read(int idx) { return 0; } -static inline int log_buf_copy(char *dest, int idx, int len) { return 0; } static inline int printk_ratelimit(void) { return 0; } static inline int __printk_ratelimit(int ratelimit_jiffies, \ int ratelimit_burst) { return 0; } diff --git a/include/linux/kprobes.h b/include/linux/kprobes.h index 1036631ff4fa..56d2d410415c 100644 --- a/include/linux/kprobes.h +++ b/include/linux/kprobes.h @@ -35,7 +35,6 @@ #include <linux/percpu.h> #include <linux/spinlock.h> #include <linux/rcupdate.h> -#include <linux/mutex.h> #ifdef CONFIG_KPROBES #include <asm/kprobes.h> @@ -202,7 +201,6 @@ static inline int init_test_probes(void) #endif /* CONFIG_KPROBES_SANITY_TEST */ extern spinlock_t kretprobe_lock; -extern struct mutex kprobe_mutex; extern int arch_prepare_kprobe(struct kprobe *p); extern void arch_arm_kprobe(struct kprobe *p); extern void arch_disarm_kprobe(struct kprobe *p); diff --git a/include/linux/list.h b/include/linux/list.h index 08cf4f651889..139ec41d9c2e 100644 --- a/include/linux/list.h +++ b/include/linux/list.h @@ -85,65 +85,6 @@ static inline void list_add_tail(struct list_head *new, struct list_head *head) } /* - * Insert a new entry between two known consecutive entries. - * - * This is only for internal list manipulation where we know - * the prev/next entries already! - */ -static inline void __list_add_rcu(struct list_head * new, - struct list_head * prev, struct list_head * next) -{ - new->next = next; - new->prev = prev; - smp_wmb(); - next->prev = new; - prev->next = new; -} - -/** - * list_add_rcu - add a new entry to rcu-protected list - * @new: new entry to be added - * @head: list head to add it after - * - * Insert a new entry after the specified head. - * This is good for implementing stacks. - * - * The caller must take whatever precautions are necessary - * (such as holding appropriate locks) to avoid racing - * with another list-mutation primitive, such as list_add_rcu() - * or list_del_rcu(), running on this same list. - * However, it is perfectly legal to run concurrently with - * the _rcu list-traversal primitives, such as - * list_for_each_entry_rcu(). - */ -static inline void list_add_rcu(struct list_head *new, struct list_head *head) -{ - __list_add_rcu(new, head, head->next); -} - -/** - * list_add_tail_rcu - add a new entry to rcu-protected list - * @new: new entry to be added - * @head: list head to add it before - * - * Insert a new entry before the specified head. - * This is useful for implementing queues. - * - * The caller must take whatever precautions are necessary - * (such as holding appropriate locks) to avoid racing - * with another list-mutation primitive, such as list_add_tail_rcu() - * or list_del_rcu(), running on this same list. - * However, it is perfectly legal to run concurrently with - * the _rcu list-traversal primitives, such as - * list_for_each_entry_rcu(). - */ -static inline void list_add_tail_rcu(struct list_head *new, - struct list_head *head) -{ - __list_add_rcu(new, head->prev, head); -} - -/* * Delete a list entry by making the prev/next entries * point to each other. * @@ -174,36 +115,6 @@ extern void list_del(struct list_head *entry); #endif /** - * list_del_rcu - deletes entry from list without re-initialization - * @entry: the element to delete from the list. - * - * Note: list_empty() on entry does not return true after this, - * the entry is in an undefined state. It is useful for RCU based - * lockfree traversal. - * - * In particular, it means that we can not poison the forward - * pointers that may still be used for walking the list. - * - * The caller must take whatever precautions are necessary - * (such as holding appropriate locks) to avoid racing - * with another list-mutation primitive, such as list_del_rcu() - * or list_add_rcu(), running on this same list. - * However, it is perfectly legal to run concurrently with - * the _rcu list-traversal primitives, such as - * list_for_each_entry_rcu(). - * - * Note that the caller is not permitted to immediately free - * the newly deleted entry. Instead, either synchronize_rcu() - * or call_rcu() must be used to defer freeing until an RCU - * grace period has elapsed. - */ -static inline void list_del_rcu(struct list_head *entry) -{ - __list_del(entry->prev, entry->next); - entry->prev = LIST_POISON2; -} - -/** * list_replace - replace old entry by new one * @old : the element to be replaced * @new : the new element to insert @@ -227,25 +138,6 @@ static inline void list_replace_init(struct list_head *old, } /** - * list_replace_rcu - replace old entry by new one - * @old : the element to be replaced - * @new : the new element to insert - * - * The @old entry will be replaced with the @new entry atomically. - * Note: @old should not be empty. - */ -static inline void list_replace_rcu(struct list_head *old, - struct list_head *new) -{ - new->next = old->next; - new->prev = old->prev; - smp_wmb(); - new->next->prev = new; - new->prev->next = new; - old->prev = LIST_POISON2; -} - -/** * list_del_init - deletes entry from list and reinitialize it. * @entry: the element to delete from the list. */ @@ -369,62 +261,6 @@ static inline void list_splice_init(struct list_head *list, } /** - * list_splice_init_rcu - splice an RCU-protected list into an existing list. - * @list: the RCU-protected list to splice - * @head: the place in the list to splice the first list into - * @sync: function to sync: synchronize_rcu(), synchronize_sched(), ... - * - * @head can be RCU-read traversed concurrently with this function. - * - * Note that this function blocks. - * - * Important note: the caller must take whatever action is necessary to - * prevent any other updates to @head. In principle, it is possible - * to modify the list as soon as sync() begins execution. - * If this sort of thing becomes necessary, an alternative version - * based on call_rcu() could be created. But only if -really- - * needed -- there is no shortage of RCU API members. - */ -static inline void list_splice_init_rcu(struct list_head *list, - struct list_head *head, - void (*sync)(void)) -{ - struct list_head *first = list->next; - struct list_head *last = list->prev; - struct list_head *at = head->next; - - if (list_empty(head)) - return; - - /* "first" and "last" tracking list, so initialize it. */ - - INIT_LIST_HEAD(list); - - /* - * At this point, the list body still points to the source list. - * Wait for any readers to finish using the list before splicing - * the list body into the new list. Any new readers will see - * an empty list. - */ - - sync(); - - /* - * Readers are finished with the source list, so perform splice. - * The order is important if the new list is global and accessible - * to concurrent RCU readers. Note that RCU readers are not - * permitted to traverse the prev pointers without excluding - * this function. - */ - - last->next = at; - smp_wmb(); - head->next = first; - first->prev = head; - at->prev = last; -} - -/** * list_entry - get the struct for this entry * @ptr: the &struct list_head pointer. * @type: the type of the struct this is embedded in. @@ -629,57 +465,6 @@ static inline void list_splice_init_rcu(struct list_head *list, &pos->member != (head); \ pos = n, n = list_entry(n->member.prev, typeof(*n), member)) -/** - * list_for_each_rcu - iterate over an rcu-protected list - * @pos: the &struct list_head to use as a loop cursor. - * @head: the head for your list. - * - * This list-traversal primitive may safely run concurrently with - * the _rcu list-mutation primitives such as list_add_rcu() - * as long as the traversal is guarded by rcu_read_lock(). - */ -#define list_for_each_rcu(pos, head) \ - for (pos = rcu_dereference((head)->next); \ - prefetch(pos->next), pos != (head); \ - pos = rcu_dereference(pos->next)) - -#define __list_for_each_rcu(pos, head) \ - for (pos = rcu_dereference((head)->next); \ - pos != (head); \ - pos = rcu_dereference(pos->next)) - -/** - * list_for_each_entry_rcu - iterate over rcu list of given type - * @pos: the type * to use as a loop cursor. - * @head: the head for your list. - * @member: the name of the list_struct within the struct. - * - * This list-traversal primitive may safely run concurrently with - * the _rcu list-mutation primitives such as list_add_rcu() - * as long as the traversal is guarded by rcu_read_lock(). - */ -#define list_for_each_entry_rcu(pos, head, member) \ - for (pos = list_entry(rcu_dereference((head)->next), typeof(*pos), member); \ - prefetch(pos->member.next), &pos->member != (head); \ - pos = list_entry(rcu_dereference(pos->member.next), typeof(*pos), member)) - - -/** - * list_for_each_continue_rcu - * @pos: the &struct list_head to use as a loop cursor. - * @head: the head for your list. - * - * Iterate over an rcu-protected list, continuing after current point. - * - * This list-traversal primitive may safely run concurrently with - * the _rcu list-mutation primitives such as list_add_rcu() - * as long as the traversal is guarded by rcu_read_lock(). - */ -#define list_for_each_continue_rcu(pos, head) \ - for ((pos) = rcu_dereference((pos)->next); \ - prefetch((pos)->next), (pos) != (head); \ - (pos) = rcu_dereference((pos)->next)) - /* * Double linked lists with a single pointer list head. * Mostly useful for hash tables where the two pointer list head is @@ -730,31 +515,6 @@ static inline void hlist_del(struct hlist_node *n) n->pprev = LIST_POISON2; } -/** - * hlist_del_rcu - deletes entry from hash list without re-initialization - * @n: the element to delete from the hash list. - * - * Note: list_unhashed() on entry does not return true after this, - * the entry is in an undefined state. It is useful for RCU based - * lockfree traversal. - * - * In particular, it means that we can not poison the forward - * pointers that may still be used for walking the hash list. - * - * The caller must take whatever precautions are necessary - * (such as holding appropriate locks) to avoid racing - * with another list-mutation primitive, such as hlist_add_head_rcu() - * or hlist_del_rcu(), running on this same list. - * However, it is perfectly legal to run concurrently with - * the _rcu list-traversal primitives, such as - * hlist_for_each_entry(). - */ -static inline void hlist_del_rcu(struct hlist_node *n) -{ - __hlist_del(n); - n->pprev = LIST_POISON2; -} - static inline void hlist_del_init(struct hlist_node *n) { if (!hlist_unhashed(n)) { @@ -763,27 +523,6 @@ static inline void hlist_del_init(struct hlist_node *n) } } -/** - * hlist_replace_rcu - replace old entry by new one - * @old : the element to be replaced - * @new : the new element to insert - * - * The @old entry will be replaced with the @new entry atomically. - */ -static inline void hlist_replace_rcu(struct hlist_node *old, - struct hlist_node *new) -{ - struct hlist_node *next = old->next; - - new->next = next; - new->pprev = old->pprev; - smp_wmb(); - if (next) - new->next->pprev = &new->next; - *new->pprev = new; - old->pprev = LIST_POISON2; -} - static inline void hlist_add_head(struct hlist_node *n, struct hlist_head *h) { struct hlist_node *first = h->first; @@ -794,38 +533,6 @@ static inline void hlist_add_head(struct hlist_node *n, struct hlist_head *h) n->pprev = &h->first; } - -/** - * hlist_add_head_rcu - * @n: the element to add to the hash list. - * @h: the list to add to. - * - * Description: - * Adds the specified element to the specified hlist, - * while permitting racing traversals. - * - * The caller must take whatever precautions are necessary - * (such as holding appropriate locks) to avoid racing - * with another list-mutation primitive, such as hlist_add_head_rcu() - * or hlist_del_rcu(), running on this same list. - * However, it is perfectly legal to run concurrently with - * the _rcu list-traversal primitives, such as - * hlist_for_each_entry_rcu(), used to prevent memory-consistency - * problems on Alpha CPUs. Regardless of the type of CPU, the - * list-traversal primitive must be guarded by rcu_read_lock(). - */ -static inline void hlist_add_head_rcu(struct hlist_node *n, - struct hlist_head *h) -{ - struct hlist_node *first = h->first; - n->next = first; - n->pprev = &h->first; - smp_wmb(); - if (first) - first->pprev = &n->next; - h->first = n; -} - /* next must be != NULL */ static inline void hlist_add_before(struct hlist_node *n, struct hlist_node *next) @@ -847,63 +554,6 @@ static inline void hlist_add_after(struct hlist_node *n, next->next->pprev = &next->next; } -/** - * hlist_add_before_rcu - * @n: the new element to add to the hash list. - * @next: the existing element to add the new element before. - * - * Description: - * Adds the specified element to the specified hlist - * before the specified node while permitting racing traversals. - * - * The caller must take whatever precautions are necessary - * (such as holding appropriate locks) to avoid racing - * with another list-mutation primitive, such as hlist_add_head_rcu() - * or hlist_del_rcu(), running on this same list. - * However, it is perfectly legal to run concurrently with - * the _rcu list-traversal primitives, such as - * hlist_for_each_entry_rcu(), used to prevent memory-consistency - * problems on Alpha CPUs. - */ -static inline void hlist_add_before_rcu(struct hlist_node *n, - struct hlist_node *next) -{ - n->pprev = next->pprev; - n->next = next; - smp_wmb(); - next->pprev = &n->next; - *(n->pprev) = n; -} - -/** - * hlist_add_after_rcu - * @prev: the existing element to add the new element after. - * @n: the new element to add to the hash list. - * - * Description: - * Adds the specified element to the specified hlist - * after the specified node while permitting racing traversals. - * - * The caller must take whatever precautions are necessary - * (such as holding appropriate locks) to avoid racing - * with another list-mutation primitive, such as hlist_add_head_rcu() - * or hlist_del_rcu(), running on this same list. - * However, it is perfectly legal to run concurrently with - * the _rcu list-traversal primitives, such as - * hlist_for_each_entry_rcu(), used to prevent memory-consistency - * problems on Alpha CPUs. - */ -static inline void hlist_add_after_rcu(struct hlist_node *prev, - struct hlist_node *n) -{ - n->next = prev->next; - n->pprev = &prev->next; - smp_wmb(); - prev->next = n; - if (n->next) - n->next->pprev = &n->next; -} - #define hlist_entry(ptr, type, member) container_of(ptr,type,member) #define hlist_for_each(pos, head) \ @@ -964,21 +614,4 @@ static inline void hlist_add_after_rcu(struct hlist_node *prev, ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \ pos = n) -/** - * hlist_for_each_entry_rcu - iterate over rcu list of given type - * @tpos: the type * to use as a loop cursor. - * @pos: the &struct hlist_node to use as a loop cursor. - * @head: the head for your list. - * @member: the name of the hlist_node within the struct. - * - * This list-traversal primitive may safely run concurrently with - * the _rcu list-mutation primitives such as hlist_add_head_rcu() - * as long as the traversal is guarded by rcu_read_lock(). - */ -#define hlist_for_each_entry_rcu(tpos, pos, head, member) \ - for (pos = rcu_dereference((head)->first); \ - pos && ({ prefetch(pos->next); 1;}) && \ - ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1;}); \ - pos = rcu_dereference(pos->next)) - #endif diff --git a/include/linux/marker.h b/include/linux/marker.h index 430f6adf9762..33f3be747d8d 100644 --- a/include/linux/marker.h +++ b/include/linux/marker.h @@ -12,6 +12,7 @@ * See the file COPYING for more details. */ +#include <linux/immediate.h> #include <linux/types.h> struct module; @@ -42,10 +43,10 @@ struct marker { const char *format; /* Marker format string, describing the * variable argument list. */ - char state; /* Marker state. */ + DEFINE_IMV(char, state);/* Immediate value state. */ char ptype; /* probe type : 0 : single, 1 : multi */ - void (*call)(const struct marker *mdata, /* Probe wrapper */ - void *call_private, const char *fmt, ...); + /* Probe wrapper */ + void (*call)(const struct marker *mdata, void *call_private, ...); struct marker_probe_closure single; struct marker_probe_closure *multi; } __attribute__((aligned(8))); @@ -58,8 +59,12 @@ struct marker { * Make sure the alignment of the structure in the __markers section will * not add unwanted padding between the beginning of the section and the * structure. Force alignment to the same alignment as the section start. + * + * The "generic" argument controls which marker enabling mechanism must be used. + * If generic is true, a variable read is used. + * If generic is false, immediate values are used. */ -#define __trace_mark(name, call_private, format, args...) \ +#define __trace_mark(generic, name, call_private, format, args...) \ do { \ static const char __mstrtab_##name[] \ __attribute__((section("__markers_strings"))) \ @@ -70,17 +75,26 @@ struct marker { 0, 0, marker_probe_cb, \ { __mark_empty_function, NULL}, NULL }; \ __mark_check_format(format, ## args); \ - if (unlikely(__mark_##name.state)) { \ - (*__mark_##name.call) \ - (&__mark_##name, call_private, \ - format, ## args); \ + if (!generic) { \ + if (unlikely(imv_cond(__mark_##name.state))) { \ + imv_cond_end(); \ + (*__mark_##name.call) \ + (&__mark_##name, call_private, \ + ## args); \ + } else \ + imv_cond_end(); \ + } else { \ + if (unlikely(_imv_read(__mark_##name.state))) \ + (*__mark_##name.call) \ + (&__mark_##name, call_private, \ + ## args); \ } \ } while (0) extern void marker_update_probe_range(struct marker *begin, struct marker *end); #else /* !CONFIG_MARKERS */ -#define __trace_mark(name, call_private, format, args...) \ +#define __trace_mark(generic, name, call_private, format, args...) \ __mark_check_format(format, ## args) static inline void marker_update_probe_range(struct marker *begin, struct marker *end) @@ -88,15 +102,30 @@ static inline void marker_update_probe_range(struct marker *begin, #endif /* CONFIG_MARKERS */ /** - * trace_mark - Marker + * trace_mark - Marker using code patching * @name: marker name, not quoted. * @format: format string * @args...: variable argument list * - * Places a marker. + * Places a marker using optimized code patching technique (imv_read()) + * to be enabled when immediate values are present. */ #define trace_mark(name, format, args...) \ - __trace_mark(name, NULL, format, ## args) + __trace_mark(0, name, NULL, format, ## args) + +/** + * _trace_mark - Marker using variable read + * @name: marker name, not quoted. + * @format: format string + * @args...: variable argument list + * + * Places a marker using a standard memory read (_imv_read()) to be + * enabled. Should be used for markers in code paths where instruction + * modification based enabling is not welcome. (__init and __exit functions, + * lockdep, some traps, printk). + */ +#define _trace_mark(name, format, args...) \ + __trace_mark(1, name, NULL, format, ## args) /** * MARK_NOARGS - Format string for a marker with no argument. @@ -117,9 +146,9 @@ static inline void __printf(1, 2) ___mark_check_format(const char *fmt, ...) extern marker_probe_func __mark_empty_function; extern void marker_probe_cb(const struct marker *mdata, - void *call_private, const char *fmt, ...); + void *call_private, ...); extern void marker_probe_cb_noarg(const struct marker *mdata, - void *call_private, const char *fmt, ...); + void *call_private, ...); /* * Connect a probe to a marker. diff --git a/include/linux/memory.h b/include/linux/memory.h index 2f5f8a5ef2a0..95379b568050 100644 --- a/include/linux/memory.h +++ b/include/linux/memory.h @@ -99,4 +99,11 @@ extern int memory_notify(unsigned long val, void *v); #define hotplug_memory_notifier(fn, pri) do { } while (0) #endif +/* + * Take and release the kernel text modification lock, used for code patching. + * Users of this lock can sleep. + */ +extern void kernel_text_lock(void); +extern void kernel_text_unlock(void); + #endif /* _LINUX_MEMORY_H_ */ diff --git a/include/linux/mm.h b/include/linux/mm.h index c31a9cd2a30e..28694deb95f8 100644 --- a/include/linux/mm.h +++ b/include/linux/mm.h @@ -1023,6 +1023,7 @@ extern void mem_init(void); extern void show_mem(void); extern void si_meminfo(struct sysinfo * val); extern void si_meminfo_node(struct sysinfo *val, int nid); +extern int after_bootmem; #ifdef CONFIG_NUMA extern void setup_per_cpu_pageset(void); diff --git a/include/linux/module.h b/include/linux/module.h index 3e03b1acbc94..3263812bf9e8 100644 --- a/include/linux/module.h +++ b/include/linux/module.h @@ -15,6 +15,7 @@ #include <linux/stringify.h> #include <linux/kobject.h> #include <linux/moduleparam.h> +#include <linux/immediate.h> #include <linux/marker.h> #include <asm/local.h> @@ -338,6 +339,12 @@ struct module /* The command line arguments (may be mangled). People like keeping pointers to this stuff */ char *args; +#ifdef CONFIG_IMMEDIATE + struct __imv *immediate; + unsigned int num_immediate; + unsigned long *immediate_cond_end; + unsigned int num_immediate_cond_end; +#endif #ifdef CONFIG_MARKERS struct marker *markers; unsigned int num_markers; @@ -556,6 +563,24 @@ static inline void module_update_markers(void) #endif /* CONFIG_MODULES */ +#if defined(CONFIG_MODULES) && defined(CONFIG_IMMEDIATE) +extern void _module_imv_update(void); +extern void module_imv_update(void); +extern int is_imv_cond_end_module(unsigned long addr1, unsigned long addr2); +#else +static inline void _module_imv_update(void) +{ +} +static inline void module_imv_update(void) +{ +} +static inline int is_imv_cond_end_module(unsigned long addr1, + unsigned long addr2) +{ + return 0; +} +#endif + struct device_driver; #ifdef CONFIG_SYSFS struct module; diff --git a/include/linux/profile.h b/include/linux/profile.h index 05c1cc736937..1397c38b85b0 100644 --- a/include/linux/profile.h +++ b/include/linux/profile.h @@ -5,10 +5,11 @@ #include <linux/init.h> #include <linux/cpumask.h> #include <linux/cache.h> +#include <linux/immediate.h> #include <asm/errno.h> -extern int prof_on __read_mostly; +DECLARE_IMV(char, prof_on) __read_mostly; #define CPU_PROFILING 1 #define SCHED_PROFILING 2 @@ -36,7 +37,7 @@ static inline void profile_hit(int type, void *ip) /* * Speedup for the common (no profiling enabled) case: */ - if (unlikely(prof_on == type)) + if (unlikely(imv_read(prof_on) == type)) profile_hits(type, ip, 1); } diff --git a/include/linux/rcuclassic.h b/include/linux/rcuclassic.h index b3aa05baab8a..8c774905dcfe 100644 --- a/include/linux/rcuclassic.h +++ b/include/linux/rcuclassic.h @@ -151,7 +151,10 @@ extern struct lockdep_map rcu_lock_map; #define __synchronize_sched() synchronize_rcu() +#define call_rcu_sched(head, func) call_rcu(head, func) + extern void __rcu_init(void); +#define rcu_init_sched() do { } while (0) extern void rcu_check_callbacks(int cpu, int user); extern void rcu_restart_cpu(int cpu); diff --git a/include/linux/rculist.h b/include/linux/rculist.h new file mode 100644 index 000000000000..b0f39be08b6c --- /dev/null +++ b/include/linux/rculist.h @@ -0,0 +1,373 @@ +#ifndef _LINUX_RCULIST_H +#define _LINUX_RCULIST_H + +#ifdef __KERNEL__ + +/* + * RCU-protected list version + */ +#include <linux/list.h> +#include <linux/rcupdate.h> + +/* + * Insert a new entry between two known consecutive entries. + * + * This is only for internal list manipulation where we know + * the prev/next entries already! + */ +static inline void __list_add_rcu(struct list_head *new, + struct list_head *prev, struct list_head *next) +{ + new->next = next; + new->prev = prev; + rcu_assign_pointer(prev->next, new); + next->prev = new; +} + +/** + * list_add_rcu - add a new entry to rcu-protected list + * @new: new entry to be added + * @head: list head to add it after + * + * Insert a new entry after the specified head. + * This is good for implementing stacks. + * + * The caller must take whatever precautions are necessary + * (such as holding appropriate locks) to avoid racing + * with another list-mutation primitive, such as list_add_rcu() + * or list_del_rcu(), running on this same list. + * However, it is perfectly legal to run concurrently with + * the _rcu list-traversal primitives, such as + * list_for_each_entry_rcu(). + */ +static inline void list_add_rcu(struct list_head *new, struct list_head *head) +{ + __list_add_rcu(new, head, head->next); +} + +/** + * list_add_tail_rcu - add a new entry to rcu-protected list + * @new: new entry to be added + * @head: list head to add it before + * + * Insert a new entry before the specified head. + * This is useful for implementing queues. + * + * The caller must take whatever precautions are necessary + * (such as holding appropriate locks) to avoid racing + * with another list-mutation primitive, such as list_add_tail_rcu() + * or list_del_rcu(), running on this same list. + * However, it is perfectly legal to run concurrently with + * the _rcu list-traversal primitives, such as + * list_for_each_entry_rcu(). + */ +static inline void list_add_tail_rcu(struct list_head *new, + struct list_head *head) +{ + __list_add_rcu(new, head->prev, head); +} + +/** + * list_del_rcu - deletes entry from list without re-initialization + * @entry: the element to delete from the list. + * + * Note: list_empty() on entry does not return true after this, + * the entry is in an undefined state. It is useful for RCU based + * lockfree traversal. + * + * In particular, it means that we can not poison the forward + * pointers that may still be used for walking the list. + * + * The caller must take whatever precautions are necessary + * (such as holding appropriate locks) to avoid racing + * with another list-mutation primitive, such as list_del_rcu() + * or list_add_rcu(), running on this same list. + * However, it is perfectly legal to run concurrently with + * the _rcu list-traversal primitives, such as + * list_for_each_entry_rcu(). + * + * Note that the caller is not permitted to immediately free + * the newly deleted entry. Instead, either synchronize_rcu() + * or call_rcu() must be used to defer freeing until an RCU + * grace period has elapsed. + */ +static inline void list_del_rcu(struct list_head *entry) +{ + __list_del(entry->prev, entry->next); + entry->prev = LIST_POISON2; +} + +/** + * list_replace_rcu - replace old entry by new one + * @old : the element to be replaced + * @new : the new element to insert + * + * The @old entry will be replaced with the @new entry atomically. + * Note: @old should not be empty. + */ +static inline void list_replace_rcu(struct list_head *old, + struct list_head *new) +{ + new->next = old->next; + new->prev = old->prev; + rcu_assign_pointer(new->prev->next, new); + new->next->prev = new; + old->prev = LIST_POISON2; +} + +/** + * list_splice_init_rcu - splice an RCU-protected list into an existing list. + * @list: the RCU-protected list to splice + * @head: the place in the list to splice the first list into + * @sync: function to sync: synchronize_rcu(), synchronize_sched(), ... + * + * @head can be RCU-read traversed concurrently with this function. + * + * Note that this function blocks. + * + * Important note: the caller must take whatever action is necessary to + * prevent any other updates to @head. In principle, it is possible + * to modify the list as soon as sync() begins execution. + * If this sort of thing becomes necessary, an alternative version + * based on call_rcu() could be created. But only if -really- + * needed -- there is no shortage of RCU API members. + */ +static inline void list_splice_init_rcu(struct list_head *list, + struct list_head *head, + void (*sync)(void)) +{ + struct list_head *first = list->next; + struct list_head *last = list->prev; + struct list_head *at = head->next; + + if (list_empty(head)) + return; + + /* "first" and "last" tracking list, so initialize it. */ + + INIT_LIST_HEAD(list); + + /* + * At this point, the list body still points to the source list. + * Wait for any readers to finish using the list before splicing + * the list body into the new list. Any new readers will see + * an empty list. + */ + + sync(); + + /* + * Readers are finished with the source list, so perform splice. + * The order is important if the new list is global and accessible + * to concurrent RCU readers. Note that RCU readers are not + * permitted to traverse the prev pointers without excluding + * this function. + */ + + last->next = at; + rcu_assign_pointer(head->next, first); + first->prev = head; + at->prev = last; +} + +/** + * list_for_each_rcu - iterate over an rcu-protected list + * @pos: the &struct list_head to use as a loop cursor. + * @head: the head for your list. + * + * This list-traversal primitive may safely run concurrently with + * the _rcu list-mutation primitives such as list_add_rcu() + * as long as the traversal is guarded by rcu_read_lock(). + */ +#define list_for_each_rcu(pos, head) \ + for (pos = rcu_dereference((head)->next); \ + prefetch(pos->next), pos != (head); \ + pos = rcu_dereference(pos->next)) + +#define __list_for_each_rcu(pos, head) \ + for (pos = rcu_dereference((head)->next); \ + pos != (head); \ + pos = rcu_dereference(pos->next)) + +/** + * list_for_each_entry_rcu - iterate over rcu list of given type + * @pos: the type * to use as a loop cursor. + * @head: the head for your list. + * @member: the name of the list_struct within the struct. + * + * This list-traversal primitive may safely run concurrently with + * the _rcu list-mutation primitives such as list_add_rcu() + * as long as the traversal is guarded by rcu_read_lock(). + */ +#define list_for_each_entry_rcu(pos, head, member) \ + for (pos = list_entry(rcu_dereference((head)->next), typeof(*pos), member); \ + prefetch(pos->member.next), &pos->member != (head); \ + pos = list_entry(rcu_dereference(pos->member.next), typeof(*pos), member)) + + +/** + * list_for_each_continue_rcu + * @pos: the &struct list_head to use as a loop cursor. + * @head: the head for your list. + * + * Iterate over an rcu-protected list, continuing after current point. + * + * This list-traversal primitive may safely run concurrently with + * the _rcu list-mutation primitives such as list_add_rcu() + * as long as the traversal is guarded by rcu_read_lock(). + */ +#define list_for_each_continue_rcu(pos, head) \ + for ((pos) = rcu_dereference((pos)->next); \ + prefetch((pos)->next), (pos) != (head); \ + (pos) = rcu_dereference((pos)->next)) + +/** + * hlist_del_rcu - deletes entry from hash list without re-initialization + * @n: the element to delete from the hash list. + * + * Note: list_unhashed() on entry does not return true after this, + * the entry is in an undefined state. It is useful for RCU based + * lockfree traversal. + * + * In particular, it means that we can not poison the forward + * pointers that may still be used for walking the hash list. + * + * The caller must take whatever precautions are necessary + * (such as holding appropriate locks) to avoid racing + * with another list-mutation primitive, such as hlist_add_head_rcu() + * or hlist_del_rcu(), running on this same list. + * However, it is perfectly legal to run concurrently with + * the _rcu list-traversal primitives, such as + * hlist_for_each_entry(). + */ +static inline void hlist_del_rcu(struct hlist_node *n) +{ + __hlist_del(n); + n->pprev = LIST_POISON2; +} + +/** + * hlist_replace_rcu - replace old entry by new one + * @old : the element to be replaced + * @new : the new element to insert + * + * The @old entry will be replaced with the @new entry atomically. + */ +static inline void hlist_replace_rcu(struct hlist_node *old, + struct hlist_node *new) +{ + struct hlist_node *next = old->next; + + new->next = next; + new->pprev = old->pprev; + rcu_assign_pointer(*new->pprev, new); + if (next) + new->next->pprev = &new->next; + old->pprev = LIST_POISON2; +} + +/** + * hlist_add_head_rcu + * @n: the element to add to the hash list. + * @h: the list to add to. + * + * Description: + * Adds the specified element to the specified hlist, + * while permitting racing traversals. + * + * The caller must take whatever precautions are necessary + * (such as holding appropriate locks) to avoid racing + * with another list-mutation primitive, such as hlist_add_head_rcu() + * or hlist_del_rcu(), running on this same list. + * However, it is perfectly legal to run concurrently with + * the _rcu list-traversal primitives, such as + * hlist_for_each_entry_rcu(), used to prevent memory-consistency + * problems on Alpha CPUs. Regardless of the type of CPU, the + * list-traversal primitive must be guarded by rcu_read_lock(). + */ +static inline void hlist_add_head_rcu(struct hlist_node *n, + struct hlist_head *h) +{ + struct hlist_node *first = h->first; + + n->next = first; + n->pprev = &h->first; + rcu_assign_pointer(h->first, n); + if (first) + first->pprev = &n->next; +} + +/** + * hlist_add_before_rcu + * @n: the new element to add to the hash list. + * @next: the existing element to add the new element before. + * + * Description: + * Adds the specified element to the specified hlist + * before the specified node while permitting racing traversals. + * + * The caller must take whatever precautions are necessary + * (such as holding appropriate locks) to avoid racing + * with another list-mutation primitive, such as hlist_add_head_rcu() + * or hlist_del_rcu(), running on this same list. + * However, it is perfectly legal to run concurrently with + * the _rcu list-traversal primitives, such as + * hlist_for_each_entry_rcu(), used to prevent memory-consistency + * problems on Alpha CPUs. + */ +static inline void hlist_add_before_rcu(struct hlist_node *n, + struct hlist_node *next) +{ + n->pprev = next->pprev; + n->next = next; + rcu_assign_pointer(*(n->pprev), n); + next->pprev = &n->next; +} + +/** + * hlist_add_after_rcu + * @prev: the existing element to add the new element after. + * @n: the new element to add to the hash list. + * + * Description: + * Adds the specified element to the specified hlist + * after the specified node while permitting racing traversals. + * + * The caller must take whatever precautions are necessary + * (such as holding appropriate locks) to avoid racing + * with another list-mutation primitive, such as hlist_add_head_rcu() + * or hlist_del_rcu(), running on this same list. + * However, it is perfectly legal to run concurrently with + * the _rcu list-traversal primitives, such as + * hlist_for_each_entry_rcu(), used to prevent memory-consistency + * problems on Alpha CPUs. + */ +static inline void hlist_add_after_rcu(struct hlist_node *prev, + struct hlist_node *n) +{ + n->next = prev->next; + n->pprev = &prev->next; + rcu_assign_pointer(prev->next, n); + if (n->next) + n->next->pprev = &n->next; +} + +/** + * hlist_for_each_entry_rcu - iterate over rcu list of given type + * @tpos: the type * to use as a loop cursor. + * @pos: the &struct hlist_node to use as a loop cursor. + * @head: the head for your list. + * @member: the name of the hlist_node within the struct. + * + * This list-traversal primitive may safely run concurrently with + * the _rcu list-mutation primitives such as hlist_add_head_rcu() + * as long as the traversal is guarded by rcu_read_lock(). + */ +#define hlist_for_each_entry_rcu(tpos, pos, head, member) \ + for (pos = rcu_dereference((head)->first); \ + pos && ({ prefetch(pos->next); 1; }) && \ + ({ tpos = hlist_entry(pos, typeof(*tpos), member); 1; }); \ + pos = rcu_dereference(pos->next)) + +#endif /* __KERNEL__ */ +#endif diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h index 8082d6587a0f..cfeed4db46a7 100644 --- a/include/linux/rcupdate.h +++ b/include/linux/rcupdate.h @@ -40,6 +40,7 @@ #include <linux/cpumask.h> #include <linux/seqlock.h> #include <linux/lockdep.h> +#include <linux/completion.h> /** * struct rcu_head - callback structure for use with RCU @@ -180,6 +181,27 @@ struct rcu_head { (p) = (v); \ }) +/* Infrastructure to implement the synchronize_() primitives. */ + +struct rcu_synchronize { + struct rcu_head head; + struct completion completion; +}; + +extern void wakeme_after_rcu(struct rcu_head *head); + +#define synchronize_rcu_xxx(name, func) \ +void name(void) \ +{ \ + struct rcu_synchronize rcu; \ + \ + init_completion(&rcu.completion); \ + /* Will wake me after RCU finished. */ \ + func(&rcu.head, wakeme_after_rcu); \ + /* Wait for it. */ \ + wait_for_completion(&rcu.completion); \ +} + /** * synchronize_sched - block until all CPUs have exited any non-preemptive * kernel code sequences. @@ -236,8 +258,8 @@ extern void call_rcu_bh(struct rcu_head *head, /* Exported common interfaces */ extern void synchronize_rcu(void); extern void rcu_barrier(void); -extern long rcu_batches_completed(void); -extern long rcu_batches_completed_bh(void); +extern void rcu_barrier_bh(void); +extern void rcu_barrier_sched(void); /* Internal to kernel */ extern void rcu_init(void); diff --git a/include/linux/rcupreempt.h b/include/linux/rcupreempt.h index 8a05c7e20bc4..f04b64eca636 100644 --- a/include/linux/rcupreempt.h +++ b/include/linux/rcupreempt.h @@ -40,10 +40,39 @@ #include <linux/cpumask.h> #include <linux/seqlock.h> -#define rcu_qsctr_inc(cpu) +struct rcu_dyntick_sched { + int dynticks; + int dynticks_snap; + int sched_qs; + int sched_qs_snap; + int sched_dynticks_snap; +}; + +DECLARE_PER_CPU(struct rcu_dyntick_sched, rcu_dyntick_sched); + +static inline void rcu_qsctr_inc(int cpu) +{ + struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu); + + rdssp->sched_qs++; +} #define rcu_bh_qsctr_inc(cpu) #define call_rcu_bh(head, rcu) call_rcu(head, rcu) +/** + * call_rcu_sched - Queue RCU callback for invocation after sched grace period. + * @head: structure to be used for queueing the RCU updates. + * @func: actual update function to be invoked after the grace period + * + * The update function will be invoked some time after a full + * synchronize_sched()-style grace period elapses, in other words after + * all currently executing preempt-disabled sections of code (including + * hardirq handlers, NMI handlers, and local_irq_save() blocks) have + * completed. + */ +extern void call_rcu_sched(struct rcu_head *head, + void (*func)(struct rcu_head *head)); + extern void __rcu_read_lock(void) __acquires(RCU); extern void __rcu_read_unlock(void) __releases(RCU); extern int rcu_pending(int cpu); @@ -55,6 +84,7 @@ extern int rcu_needs_cpu(int cpu); extern void __synchronize_sched(void); extern void __rcu_init(void); +extern void rcu_init_sched(void); extern void rcu_check_callbacks(int cpu, int user); extern void rcu_restart_cpu(int cpu); extern long rcu_batches_completed(void); @@ -81,20 +111,20 @@ extern struct rcupreempt_trace *rcupreempt_trace_cpu(int cpu); struct softirq_action; #ifdef CONFIG_NO_HZ -DECLARE_PER_CPU(long, dynticks_progress_counter); +DECLARE_PER_CPU(struct rcu_dyntick_sched, rcu_dyntick_sched); static inline void rcu_enter_nohz(void) { smp_mb(); /* CPUs seeing ++ must see prior RCU read-side crit sects */ - __get_cpu_var(dynticks_progress_counter)++; - WARN_ON(__get_cpu_var(dynticks_progress_counter) & 0x1); + __get_cpu_var(rcu_dyntick_sched).dynticks++; + WARN_ON(__get_cpu_var(rcu_dyntick_sched).dynticks & 0x1); } static inline void rcu_exit_nohz(void) { - __get_cpu_var(dynticks_progress_counter)++; smp_mb(); /* CPUs seeing ++ must see later RCU read-side crit sects */ - WARN_ON(!(__get_cpu_var(dynticks_progress_counter) & 0x1)); + __get_cpu_var(rcu_dyntick_sched).dynticks++; + WARN_ON(!(__get_cpu_var(rcu_dyntick_sched).dynticks & 0x1)); } #else /* CONFIG_NO_HZ */ diff --git a/include/linux/sched.h b/include/linux/sched.h index 41f3c50e29ee..d3248cd56cbf 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -134,7 +134,6 @@ extern unsigned long nr_running(void); extern unsigned long nr_uninterruptible(void); extern unsigned long nr_active(void); extern unsigned long nr_iowait(void); -extern unsigned long weighted_cpuload(const int cpu); struct seq_file; struct cfs_rq; @@ -296,10 +295,11 @@ extern void softlockup_tick(void); extern void spawn_softlockup_task(void); extern void touch_softlockup_watchdog(void); extern void touch_all_softlockup_watchdogs(void); -extern unsigned long softlockup_thresh; +extern unsigned int softlockup_panic; extern unsigned long sysctl_hung_task_check_count; extern unsigned long sysctl_hung_task_timeout_secs; extern unsigned long sysctl_hung_task_warnings; +extern int softlockup_thresh; #else static inline void softlockup_tick(void) { @@ -826,23 +826,6 @@ extern int arch_reinit_sched_domains(void); #endif /* CONFIG_SMP */ -/* - * A runqueue laden with a single nice 0 task scores a weighted_cpuload of - * SCHED_LOAD_SCALE. This function returns 1 if any cpu is laden with a - * task of nice 0 or enough lower priority tasks to bring up the - * weighted_cpuload - */ -static inline int above_background_load(void) -{ - unsigned long cpu; - - for_each_online_cpu(cpu) { - if (weighted_cpuload(cpu) >= SCHED_LOAD_SCALE) - return 1; - } - return 0; -} - struct io_context; /* See blkdev.h */ #define NGROUPS_SMALL 32 #define NGROUPS_PER_BLOCK ((unsigned int)(PAGE_SIZE / sizeof(gid_t))) @@ -933,10 +916,6 @@ struct sched_class { int running); void (*prio_changed) (struct rq *this_rq, struct task_struct *task, int oldprio, int running); - -#ifdef CONFIG_FAIR_GROUP_SCHED - void (*moved_group) (struct task_struct *p); -#endif }; struct load_weight { @@ -955,9 +934,15 @@ struct load_weight { */ struct sched_entity { struct load_weight load; /* for load-balancing */ - struct rb_node run_node; - struct list_head group_node; + struct rb_node timeline_node; +#ifdef CONFIG_FAIR_GROUP_SCHED + struct rb_node deadline_node; + u64 deadline; + u64 min_deadline; + unsigned int eligible; +#endif unsigned int on_rq; + struct list_head group_node; u64 exec_start; u64 sum_exec_runtime; @@ -2139,38 +2124,6 @@ __trace_special(void *__tr, void *__data, } #endif -#ifdef CONFIG_CONTEXT_SWITCH_TRACER -extern void -ftrace_ctx_switch(void *rq, struct task_struct *prev, struct task_struct *next); -extern void -ftrace_wake_up_task(void *rq, struct task_struct *wakee, - struct task_struct *curr); -extern void ftrace_all_fair_tasks(void *__rq, void *__tr, void *__data); -extern void -ftrace_special(unsigned long arg1, unsigned long arg2, unsigned long arg3); -#else -static inline void -ftrace_ctx_switch(void *rq, struct task_struct *prev, struct task_struct *next) -{ -} -static inline void -sched_trace_special(unsigned long p1, unsigned long p2, unsigned long p3) -{ -} -static inline void -ftrace_wake_up_task(void *rq, struct task_struct *wakee, - struct task_struct *curr) -{ -} -static inline void ftrace_all_fair_tasks(void *__rq, void *__tr, void *__data) -{ -} -static inline void -ftrace_special(unsigned long arg1, unsigned long arg2, unsigned long arg3) -{ -} -#endif - extern long sched_setaffinity(pid_t pid, const cpumask_t *new_mask); extern long sched_getaffinity(pid_t pid, cpumask_t *mask); diff --git a/include/linux/stringify.h b/include/linux/stringify.h index 0b4388356c87..de04a96c50de 100644 --- a/include/linux/stringify.h +++ b/include/linux/stringify.h @@ -6,7 +6,12 @@ * converts to "bar". */ +#ifdef __STDC__ +#define __stringify_1(x...) #x +#define __stringify(x...) __stringify_1(x) +#else /* Support gcc -traditional, without commas. */ #define __stringify_1(x) #x #define __stringify(x) __stringify_1(x) +#endif #endif /* !__LINUX_STRINGIFY_H */ diff --git a/include/linux/swapops.h b/include/linux/swapops.h index 6ec39ab27b4b..fb40dace57ec 100644 --- a/include/linux/swapops.h +++ b/include/linux/swapops.h @@ -76,6 +76,14 @@ static inline pte_t swp_entry_to_pte(swp_entry_t entry) return __swp_entry_to_pte(arch_entry); } +static inline swp_entry_t page_swp_entry(struct page *page) +{ + swp_entry_t entry; + VM_BUG_ON(!PageSwapCache(page)); + entry.val = page_private(page); + return entry; +} + #ifdef CONFIG_MIGRATION static inline swp_entry_t make_migration_entry(struct page *page, int write) { diff --git a/include/linux/writeback.h b/include/linux/writeback.h index f462439cc288..bd91987c065f 100644 --- a/include/linux/writeback.h +++ b/include/linux/writeback.h @@ -105,6 +105,8 @@ extern int vm_highmem_is_dirtyable; extern int block_dump; extern int laptop_mode; +extern unsigned long determine_dirtyable_memory(void); + extern int dirty_ratio_handler(struct ctl_table *table, int write, struct file *filp, void __user *buffer, size_t *lenp, loff_t *ppos); diff --git a/init/Kconfig b/init/Kconfig index 4c33316743f5..a0dd9c0a795c 100644 --- a/init/Kconfig +++ b/init/Kconfig @@ -791,6 +791,24 @@ config PROC_PAGE_MONITOR /proc/kpagecount, and /proc/kpageflags. Disabling these interfaces will reduce the size of the kernel by approximately 4kb. +config HAVE_IMMEDIATE + def_bool n + +config IMMEDIATE + default y + depends on HAVE_IMMEDIATE + bool "Immediate value optimization" if EMBEDDED + help + Immediate values are used as read-mostly variables that are rarely + updated. They use code patching to modify the values inscribed in the + instruction stream. It provides a way to save precious cache lines + that would otherwise have to be used by these variables. They can be + disabled through the EMBEDDED menu. + + It consumes slightly more memory and modifies the instruction stream + each time any specially-marked variable is updated. Should really be + disabled for embedded systems with read-only text. + endmenu # General setup config SLABINFO diff --git a/init/main.c b/init/main.c index 5c6c01b35113..fd699bc15c0b 100644 --- a/init/main.c +++ b/init/main.c @@ -62,6 +62,7 @@ #include <linux/signal.h> #include <linux/idr.h> #include <linux/kmemcheck.h> +#include <linux/immediate.h> #include <asm/io.h> #include <asm/bugs.h> @@ -105,6 +106,11 @@ static inline void mark_rodata_ro(void) { } #ifdef CONFIG_TC extern void tc_init(void); #endif +#ifdef CONFIG_IMMEDIATE +extern void imv_init_complete(void); +#else +static inline void imv_init_complete(void) { } +#endif enum system_states system_state; EXPORT_SYMBOL(system_state); @@ -554,6 +560,7 @@ asmlinkage void __init start_kernel(void) boot_init_stack_canary(); cgroup_init_early(); + core_imv_update(); local_irq_disable(); early_boot_irqs_off(); @@ -683,6 +690,7 @@ asmlinkage void __init start_kernel(void) cpuset_init(); taskstats_init_early(); delayacct_init(); + imv_init_complete(); check_bugs(); @@ -764,6 +772,7 @@ static void __init do_initcalls(void) */ static void __init do_basic_setup(void) { + rcu_init_sched(); /* needed by module_init stage. */ /* drivers will send hotplug events */ init_workqueues(); usermodehelper_init(); @@ -803,6 +812,7 @@ static void run_init_process(char *init_filename) */ static int noinline init_post(void) { + imv_unref_core_init(); free_initmem(); unlock_kernel(); mark_rodata_ro(); diff --git a/ipc/msg.c b/ipc/msg.c index 32494e8cc7a5..66bab29a3483 100644 --- a/ipc/msg.c +++ b/ipc/msg.c @@ -38,6 +38,7 @@ #include <linux/rwsem.h> #include <linux/nsproxy.h> #include <linux/ipc_namespace.h> +#include <linux/marker.h> #include <asm/current.h> #include <asm/uaccess.h> @@ -315,6 +316,7 @@ asmlinkage long sys_msgget(key_t key, int msgflg) struct ipc_namespace *ns; struct ipc_ops msg_ops; struct ipc_params msg_params; + long ret; ns = current->nsproxy->ipc_ns; @@ -325,7 +327,9 @@ asmlinkage long sys_msgget(key_t key, int msgflg) msg_params.key = key; msg_params.flg = msgflg; - return ipcget(ns, &msg_ids(ns), &msg_ops, &msg_params); + ret = ipcget(ns, &msg_ids(ns), &msg_ops, &msg_params); + trace_mark(ipc_msg_create, "id %ld flags %d", ret, msgflg); + return ret; } static inline unsigned long diff --git a/ipc/sem.c b/ipc/sem.c index e9418df5ff3e..705d87fa4573 100644 --- a/ipc/sem.c +++ b/ipc/sem.c @@ -83,6 +83,7 @@ #include <linux/rwsem.h> #include <linux/nsproxy.h> #include <linux/ipc_namespace.h> +#include <linux/marker.h> #include <asm/uaccess.h> #include "util.h" @@ -314,6 +315,7 @@ asmlinkage long sys_semget(key_t key, int nsems, int semflg) struct ipc_namespace *ns; struct ipc_ops sem_ops; struct ipc_params sem_params; + long err; ns = current->nsproxy->ipc_ns; @@ -328,7 +330,9 @@ asmlinkage long sys_semget(key_t key, int nsems, int semflg) sem_params.flg = semflg; sem_params.u.nsems = nsems; - return ipcget(ns, &sem_ids(ns), &sem_ops, &sem_params); + err = ipcget(ns, &sem_ids(ns), &sem_ops, &sem_params); + trace_mark(ipc_sem_create, "id %ld flags %d", err, semflg); + return err; } /* Manage the doubly linked list sma->sem_pending as a FIFO: diff --git a/ipc/shm.c b/ipc/shm.c index 554429ade079..348876073a87 100644 --- a/ipc/shm.c +++ b/ipc/shm.c @@ -39,6 +39,7 @@ #include <linux/nsproxy.h> #include <linux/mount.h> #include <linux/ipc_namespace.h> +#include <linux/marker.h> #include <asm/uaccess.h> @@ -460,6 +461,7 @@ asmlinkage long sys_shmget (key_t key, size_t size, int shmflg) struct ipc_namespace *ns; struct ipc_ops shm_ops; struct ipc_params shm_params; + long err; ns = current->nsproxy->ipc_ns; @@ -471,7 +473,9 @@ asmlinkage long sys_shmget (key_t key, size_t size, int shmflg) shm_params.flg = shmflg; shm_params.u.size = size; - return ipcget(ns, &shm_ids(ns), &shm_ops, &shm_params); + err = ipcget(ns, &shm_ids(ns), &shm_ops, &shm_params); + trace_mark(ipc_shm_create, "id %ld flags %d", err, shmflg); + return err; } static inline unsigned long copy_shmid_to_user(void __user *buf, struct shmid64_ds *in, int version) diff --git a/kernel/Makefile b/kernel/Makefile index 3392de585386..0260e749c2bd 100644 --- a/kernel/Makefile +++ b/kernel/Makefile @@ -14,7 +14,7 @@ obj-y = sched.o fork.o exec_domain.o panic.o printk.o profile.o \ ifdef CONFIG_FTRACE # Do not profile debug utilities ORIG_CFLAGS := $(KBUILD_CFLAGS) -KBUILD_CFLAGS = $(if $(filter-out lockdep% %debug,$(basename $(notdir $@))), \ +KBUILD_CFLAGS = $(if $(filter-out lockdep% %debug sched_clock,$(basename $(notdir $@))), \ $(ORIG_CFLAGS), \ $(subst -pg,,$(ORIG_CFLAGS))) endif @@ -71,6 +71,7 @@ obj-$(CONFIG_RELAY) += relay.o obj-$(CONFIG_SYSCTL) += utsname_sysctl.o obj-$(CONFIG_TASK_DELAY_ACCT) += delayacct.o obj-$(CONFIG_TASKSTATS) += taskstats.o tsacct.o +obj-$(CONFIG_IMMEDIATE) += immediate.o obj-$(CONFIG_MARKERS) += marker.o obj-$(CONFIG_LATENCYTOP) += latencytop.o obj-$(CONFIG_FTRACE) += trace/ diff --git a/kernel/cpu.c b/kernel/cpu.c index a40b4f263149..c284b64bec82 100644 --- a/kernel/cpu.c +++ b/kernel/cpu.c @@ -391,7 +391,7 @@ void __ref enable_nonboot_cpus(void) goto out; printk("Enabling non-boot CPUs ...\n"); - for_each_cpu_mask(cpu, frozen_cpus) { + for_each_cpu_mask_nr(cpu, frozen_cpus) { error = _cpu_up(cpu, 1); if (!error) { printk("CPU%d is up\n", cpu); diff --git a/kernel/cpuset.c b/kernel/cpuset.c index 8da627d33804..1a9d4ab2a558 100644 --- a/kernel/cpuset.c +++ b/kernel/cpuset.c @@ -64,6 +64,7 @@ * short circuit some hooks. */ int number_of_cpusets __read_mostly; +int number_of_system_sets __read_mostly; /* Forward declare cgroup structures */ struct cgroup_subsys cpuset_subsys; @@ -132,6 +133,7 @@ typedef enum { CS_SCHED_LOAD_BALANCE, CS_SPREAD_PAGE, CS_SPREAD_SLAB, + CS_SYSTEM, } cpuset_flagbits_t; /* convenient tests for these bits */ @@ -170,6 +172,11 @@ static inline int is_spread_slab(const struct cpuset *cs) return test_bit(CS_SPREAD_SLAB, &cs->flags); } +static inline int is_system(const struct cpuset *cs) +{ + return test_bit(CS_SYSTEM, &cs->flags); +} + /* * Increment this integer everytime any cpuset changes its * mems_allowed value. Users of cpusets can track this generation @@ -192,7 +199,9 @@ static inline int is_spread_slab(const struct cpuset *cs) static int cpuset_mems_generation; static struct cpuset top_cpuset = { - .flags = ((1 << CS_CPU_EXCLUSIVE) | (1 << CS_MEM_EXCLUSIVE)), + .flags = ((1 << CS_CPU_EXCLUSIVE) | + (1 << CS_MEM_EXCLUSIVE) | + (1 << CS_SYSTEM)), .cpus_allowed = CPU_MASK_ALL, .mems_allowed = NODE_MASK_ALL, }; @@ -474,6 +483,9 @@ static int validate_change(const struct cpuset *cur, const struct cpuset *trial) } } + if (number_of_system_sets == 1 && is_system(cur) && !is_system(trial)) + return -EINVAL; + return 0; } @@ -1046,6 +1058,74 @@ static int update_relax_domain_level(struct cpuset *cs, char *buf) return 0; } +BLOCKING_NOTIFIER_HEAD(system_map_notifier); +EXPORT_SYMBOL_GPL(system_map_notifier); + +int cpus_match_system(cpumask_t mask) +{ + cpumask_t online_system, online_mask; + + cpus_and(online_system, cpu_system_map, cpu_online_map); + cpus_and(online_mask, mask, cpu_online_map); + + return cpus_equal(online_system, online_mask); +} + +static void rebuild_system_map(void) +{ + cpumask_t *new_system_map; + struct kfifo *q = NULL; + struct cpuset *cp; + + new_system_map = kmalloc(sizeof(cpumask_t), GFP_KERNEL); + if (!new_system_map) + return; + + if (is_system(&top_cpuset)) { + cpus_setall(*new_system_map); + goto notify; + } + + cpus_clear(*new_system_map); + + q = kfifo_alloc(number_of_cpusets * sizeof(cp), GFP_KERNEL, NULL); + if (IS_ERR(q)) + goto done; + + cp = &top_cpuset; + __kfifo_put(q, (void *)&cp, sizeof(cp)); + while (__kfifo_get(q, (void *)&cp, sizeof(cp))) { + struct cgroup *cont; + struct cpuset *child; + + if (is_system(cp)) { + cpus_or(*new_system_map, + *new_system_map, cp->cpus_allowed); + continue; + } + + list_for_each_entry(cont, &cp->css.cgroup->children, sibling) { + child = cgroup_cs(cont); + __kfifo_put(q, (void *)&child, sizeof(cp)); + } + } + + if (cpus_empty(*new_system_map)) + BUG(); + +notify: + if (!cpus_match_system(*new_system_map)) { + blocking_notifier_call_chain(&system_map_notifier, 0, + new_system_map); + } + cpu_system_map = *new_system_map; + +done: + kfree(new_system_map); + if (q && !IS_ERR(q)) + kfifo_free(q); +} + /* * update_flag - read a 0 or a 1 in a file and update associated flag * bit: the bit to update (see cpuset_flagbits_t) @@ -1061,6 +1141,7 @@ static int update_flag(cpuset_flagbits_t bit, struct cpuset *cs, struct cpuset trialcs; int err; int cpus_nonempty, balance_flag_changed; + int system_flag_changed; trialcs = *cs; if (turning_on) @@ -1075,6 +1156,7 @@ static int update_flag(cpuset_flagbits_t bit, struct cpuset *cs, cpus_nonempty = !cpus_empty(trialcs.cpus_allowed); balance_flag_changed = (is_sched_load_balance(cs) != is_sched_load_balance(&trialcs)); + system_flag_changed = (is_system(cs) != is_system(&trialcs)); mutex_lock(&callback_mutex); cs->flags = trialcs.flags; @@ -1083,6 +1165,15 @@ static int update_flag(cpuset_flagbits_t bit, struct cpuset *cs, if (cpus_nonempty && balance_flag_changed) rebuild_sched_domains(); + if (system_flag_changed) { + rebuild_system_map(); + + if (is_system(cs)) + number_of_system_sets++; + else + number_of_system_sets--; + } + return 0; } @@ -1238,6 +1329,7 @@ typedef enum { FILE_MEMORY_PRESSURE, FILE_SPREAD_PAGE, FILE_SPREAD_SLAB, + FILE_SYSTEM, } cpuset_filetype_t; static ssize_t cpuset_common_file_write(struct cgroup *cont, @@ -1283,6 +1375,9 @@ static ssize_t cpuset_common_file_write(struct cgroup *cont, case FILE_SCHED_RELAX_DOMAIN_LEVEL: retval = update_relax_domain_level(cs, buffer); break; + case FILE_SYSTEM: + retval = update_flag(CS_SYSTEM, cs, buffer); + break; default: retval = -EINVAL; goto out2; @@ -1409,6 +1504,9 @@ static ssize_t cpuset_common_file_read(struct cgroup *cont, case FILE_SCHED_RELAX_DOMAIN_LEVEL: s += sprintf(s, "%d", cs->relax_domain_level); break; + case FILE_SYSTEM: + *s++ = is_system(cs) ? '1' : '0'; + break; default: retval = -EINVAL; goto out; @@ -1540,6 +1638,13 @@ static struct cftype cft_memory_pressure_enabled = { .private = FILE_MEMORY_PRESSURE_ENABLED, }; +static struct cftype cft_system = { + .name = "system", + .read = cpuset_common_file_read, + .write = cpuset_common_file_write, + .private = FILE_SYSTEM, +}; + static int cpuset_populate(struct cgroup_subsys *ss, struct cgroup *cont) { int err; @@ -1547,6 +1652,8 @@ static int cpuset_populate(struct cgroup_subsys *ss, struct cgroup *cont) err = cgroup_add_files(cont, ss, files, ARRAY_SIZE(files)); if (err) return err; + if ((err = cgroup_add_file(cont, ss, &cft_system)) < 0) + return err; /* memory_pressure_enabled is in root cpuset only */ if (!cont->parent) err = cgroup_add_file(cont, ss, @@ -1621,6 +1728,7 @@ static struct cgroup_subsys_state *cpuset_create( if (is_spread_slab(parent)) set_bit(CS_SPREAD_SLAB, &cs->flags); set_bit(CS_SCHED_LOAD_BALANCE, &cs->flags); + set_bit(CS_SYSTEM, &cs->flags); cpus_clear(cs->cpus_allowed); nodes_clear(cs->mems_allowed); cs->mems_generation = cpuset_mems_generation++; @@ -1629,6 +1737,7 @@ static struct cgroup_subsys_state *cpuset_create( cs->parent = parent; number_of_cpusets++; + number_of_system_sets++; return &cs->css ; } @@ -1652,8 +1761,11 @@ static void cpuset_destroy(struct cgroup_subsys *ss, struct cgroup *cont) if (is_sched_load_balance(cs)) update_flag(CS_SCHED_LOAD_BALANCE, cs, 0); + if (!is_system(cs)) + update_flag(CS_SYSTEM, cs, "1"); number_of_cpusets--; + number_of_system_sets--; kfree(cs); } @@ -1705,6 +1817,7 @@ int __init cpuset_init(void) return err; number_of_cpusets = 1; + number_of_system_sets = 1; return 0; } diff --git a/kernel/exit.c b/kernel/exit.c index 2faa3a158932..2e5a87c7ebd4 100644 --- a/kernel/exit.c +++ b/kernel/exit.c @@ -45,6 +45,7 @@ #include <linux/resource.h> #include <linux/blkdev.h> #include <linux/task_io_accounting_ops.h> +#include <linux/marker.h> #include <asm/uaccess.h> #include <asm/unistd.h> @@ -143,6 +144,8 @@ static void __exit_signal(struct task_struct *tsk) static void delayed_put_task_struct(struct rcu_head *rhp) { + trace_mark(kernel_process_free, "pid %d", + container_of(rhp, struct task_struct, rcu)->pid); put_task_struct(container_of(rhp, struct task_struct, rcu)); } @@ -1034,6 +1037,9 @@ NORET_TYPE void do_exit(long code) if (group_dead) acct_process(); + + trace_mark(kernel_process_exit, "pid %d", tsk->pid); + exit_sem(tsk); exit_files(tsk); exit_fs(tsk); @@ -1518,6 +1524,8 @@ static long do_wait(enum pid_type type, struct pid *pid, int options, struct task_struct *tsk; int flag, retval; + trace_mark(kernel_process_wait, "pid %d", pid_nr(pid)); + add_wait_queue(¤t->signal->wait_chldexit,&wait); repeat: /* If there is nothing that can match our critier just get out */ diff --git a/kernel/fork.c b/kernel/fork.c index 188883d9bd0d..ed46efc3376e 100644 --- a/kernel/fork.c +++ b/kernel/fork.c @@ -55,6 +55,7 @@ #include <linux/proc_fs.h> #include <linux/blkdev.h> #include <linux/magic.h> +#include <linux/marker.h> #include <asm/pgtable.h> #include <asm/pgalloc.h> @@ -1491,6 +1492,10 @@ long do_fork(unsigned long clone_flags, if (!IS_ERR(p)) { struct completion vfork; + trace_mark(kernel_process_fork, + "parent_pid %d child_pid %d child_tgid %d", + current->pid, p->pid, p->tgid); + nr = task_pid_vnr(p); if (clone_flags & CLONE_PARENT_SETTID) diff --git a/kernel/immediate.c b/kernel/immediate.c new file mode 100644 index 000000000000..3668a7f9d2e8 --- /dev/null +++ b/kernel/immediate.c @@ -0,0 +1,147 @@ +/* + * Copyright (C) 2007 Mathieu Desnoyers + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +#include <linux/module.h> +#include <linux/mutex.h> +#include <linux/immediate.h> +#include <linux/memory.h> + +#include <asm/sections.h> + +/* + * Kernel ready to execute the SMP update that may depend on trap and ipi. + */ +static int imv_early_boot_complete; + +extern struct __imv __start___imv[]; +extern struct __imv __stop___imv[]; +extern unsigned long __start___imv_cond_end[]; +extern unsigned long __stop___imv_cond_end[]; + +/* + * imv_mutex nests inside module_mutex. imv_mutex protects builtin + * immediates and module immediates. + */ +static DEFINE_MUTEX(imv_mutex); + +/** + * imv_update_range - Update immediate values in a range + * @begin: pointer to the beginning of the range + * @end: pointer to the end of the range + * + * Updates a range of immediates. + */ +void imv_update_range(struct __imv *begin, + struct __imv *end) +{ + struct __imv *iter; + int ret; + for (iter = begin; iter < end; iter++) { + mutex_lock(&imv_mutex); + if (!iter->imv) /* Skip removed __init immediate values */ + goto skip; + kernel_text_lock(); + ret = arch_imv_update(iter, !imv_early_boot_complete); + kernel_text_unlock(); + if (imv_early_boot_complete && ret) + printk(KERN_WARNING + "Invalid immediate value. " + "Variable at %p, " + "instruction at %p, size %hu\n", + (void *)iter->imv, + (void *)iter->var, iter->size); +skip: + mutex_unlock(&imv_mutex); + } +} +EXPORT_SYMBOL_GPL(imv_update_range); + +/** + * imv_update - update all immediate values in the kernel + * + * Iterate on the kernel core and modules to update the immediate values. + */ +void core_imv_update(void) +{ + /* Core kernel imvs */ + imv_update_range(__start___imv, __stop___imv); +} +EXPORT_SYMBOL_GPL(core_imv_update); + +/** + * imv_unref + * + * Deactivate any immediate value reference pointing into the code region in the + * range start to start + size. + */ +void imv_unref(struct __imv *begin, struct __imv *end, void *start, + unsigned long size) +{ + struct __imv *iter; + + for (iter = begin; iter < end; iter++) + if (iter->imv >= (unsigned long)start + && iter->imv < (unsigned long)start + size) + iter->imv = 0UL; +} + +void imv_unref_core_init(void) +{ + imv_unref(__start___imv, __stop___imv, __init_begin, + (unsigned long)__init_end - (unsigned long)__init_begin); +} + +int _is_imv_cond_end(unsigned long *begin, unsigned long *end, + unsigned long addr1, unsigned long addr2) +{ + unsigned long *iter; + int found = 0; + + for (iter = begin; iter < end; iter++) { + if (*iter == addr1) /* deals with addr1 == addr2 */ + found++; + if (*iter == addr2) + found++; + if (found == 2) + return 1; + } + return 0; +} + +/** + * is_imv_cond_end + * + * Check if the two given addresses are located in the immediate value condition + * end table. Addresses should be in the same object. + * The module mutex should be held when calling this function for non-core + * addresses. + */ +int is_imv_cond_end(unsigned long addr1, unsigned long addr2) +{ + if (core_kernel_text(addr1)) { + return _is_imv_cond_end(__start___imv_cond_end, + __stop___imv_cond_end, addr1, addr2); + } else { + return is_imv_cond_end_module(addr1, addr2); + } + return 0; +} + +void __init imv_init_complete(void) +{ + imv_early_boot_complete = 1; +} diff --git a/kernel/irq/handle.c b/kernel/irq/handle.c index 5fa6198e9139..80bc21f34c1c 100644 --- a/kernel/irq/handle.c +++ b/kernel/irq/handle.c @@ -15,6 +15,7 @@ #include <linux/random.h> #include <linux/interrupt.h> #include <linux/kernel_stat.h> +#include <linux/marker.h> #include "internals.h" @@ -130,6 +131,10 @@ irqreturn_t handle_IRQ_event(unsigned int irq, struct irqaction *action) { irqreturn_t ret, retval = IRQ_NONE; unsigned int status = 0; + struct pt_regs *regs = get_irq_regs(); + + trace_mark(kernel_irq_entry, "irq_id %u kernel_mode %u", irq, + (regs)?(!user_mode(regs)):(1)); handle_dynamic_tick(action); @@ -148,6 +153,8 @@ irqreturn_t handle_IRQ_event(unsigned int irq, struct irqaction *action) add_interrupt_randomness(irq); local_irq_disable(); + trace_mark(kernel_irq_exit, MARK_NOARGS); + return retval; } diff --git a/kernel/irq/manage.c b/kernel/irq/manage.c index 46d6611a33bb..163329f69d08 100644 --- a/kernel/irq/manage.c +++ b/kernel/irq/manage.c @@ -12,6 +12,8 @@ #include <linux/random.h> #include <linux/interrupt.h> #include <linux/slab.h> +#include <linux/cpumask.h> +#include <linux/cpuset.h> #include "internals.h" @@ -504,6 +506,24 @@ void free_irq(unsigned int irq, void *dev_id) } EXPORT_SYMBOL(free_irq); +#if !defined(CONFIG_AUTO_IRQ_AFFINITY) && defined(CONFIG_SMP) +int select_smp_affinity(unsigned int irq) +{ + cpumask_t online_system; + + if (!irq_can_set_affinity(irq)) + return 0; + + cpus_and(online_system, cpu_system_map, cpu_online_map); + + set_balance_irq_affinity(irq, online_system); + + irq_desc[irq].affinity = online_system; + irq_desc[irq].chip->set_affinity(irq, online_system); + return 0; +} +#endif + /** * request_irq - allocate an interrupt line * @irq: Interrupt line to allocate @@ -571,7 +591,9 @@ int request_irq(unsigned int irq, irq_handler_t handler, action->next = NULL; action->dev_id = dev_id; +#if !defined(CONFIG_AUTO_IRQ_AFFINITY) && defined(CONFIG_SMP) select_smp_affinity(irq); +#endif #ifdef CONFIG_DEBUG_SHIRQ if (irqflags & IRQF_SHARED) { @@ -596,3 +618,46 @@ int request_irq(unsigned int irq, irq_handler_t handler, return retval; } EXPORT_SYMBOL(request_irq); + +#ifdef CONFIG_CPUSETS +static int system_irq_notifier(struct notifier_block *nb, + unsigned long action, void *cpus) +{ + cpumask_t *new_system_map = (cpumask_t *)cpus; + int i; + + for (i = 0; i < NR_IRQS; i++) { + struct irq_desc *desc = &irq_desc[i]; + + if (desc->chip == &no_irq_chip || !irq_can_set_affinity(i)) + continue; + + if (cpus_match_system(desc->affinity)) { + cpumask_t online_system; + + cpus_and(online_system, *new_system_map, + cpu_online_map); + + set_balance_irq_affinity(i, online_system); + + desc->affinity = online_system; + desc->chip->set_affinity(i, online_system); + } + } + + return NOTIFY_OK; +} + +static struct notifier_block fn_system_irq_notifier = { + .notifier_call = system_irq_notifier, +}; + +static int __init init_irq(void) +{ + blocking_notifier_chain_register(&system_map_notifier, + &fn_system_irq_notifier); + return 0; +} + +module_init(init_irq); +#endif diff --git a/kernel/itimer.c b/kernel/itimer.c index ab982747d9bd..8360c4c8b06a 100644 --- a/kernel/itimer.c +++ b/kernel/itimer.c @@ -12,6 +12,7 @@ #include <linux/time.h> #include <linux/posix-timers.h> #include <linux/hrtimer.h> +#include <linux/marker.h> #include <asm/uaccess.h> @@ -132,6 +133,9 @@ enum hrtimer_restart it_real_fn(struct hrtimer *timer) struct signal_struct *sig = container_of(timer, struct signal_struct, real_timer); + trace_mark(kernel_timer_itimer_expired, "pid %d", + pid_nr(sig->leader_pid)); + kill_pid_info(SIGALRM, SEND_SIG_PRIV, sig->leader_pid); return HRTIMER_NORESTART; @@ -157,6 +161,15 @@ int do_setitimer(int which, struct itimerval *value, struct itimerval *ovalue) !timeval_valid(&value->it_interval)) return -EINVAL; + trace_mark(kernel_timer_itimer_set, + "which %d interval_sec %ld interval_usec %ld " + "value_sec %ld value_usec %ld", + which, + value->it_interval.tv_sec, + value->it_interval.tv_usec, + value->it_value.tv_sec, + value->it_value.tv_usec); + switch (which) { case ITIMER_REAL: again: diff --git a/kernel/kprobes.c b/kernel/kprobes.c index 1e0250cb9486..377594654307 100644 --- a/kernel/kprobes.c +++ b/kernel/kprobes.c @@ -43,6 +43,7 @@ #include <linux/seq_file.h> #include <linux/debugfs.h> #include <linux/kdebug.h> +#include <linux/memory.h> #include <asm-generic/sections.h> #include <asm/cacheflush.h> @@ -68,7 +69,7 @@ static struct hlist_head kretprobe_inst_table[KPROBE_TABLE_SIZE]; /* NOTE: change this value only with kprobe_mutex held */ static bool kprobe_enabled; -DEFINE_MUTEX(kprobe_mutex); /* Protects kprobe_table */ +static DEFINE_MUTEX(kprobe_mutex); /* Protects kprobe_table */ DEFINE_SPINLOCK(kretprobe_lock); /* Protects kretprobe_inst_table */ static DEFINE_PER_CPU(struct kprobe *, kprobe_instance) = NULL; @@ -107,6 +108,10 @@ enum kprobe_slot_state { SLOT_USED = 2, }; +/* + * Protects the kprobe_insn_pages list. Can nest into kprobe_mutex. + */ +static DEFINE_MUTEX(kprobe_insn_mutex); static struct hlist_head kprobe_insn_pages; static int kprobe_garbage_slots; static int collect_garbage_slots(void); @@ -143,7 +148,9 @@ kprobe_opcode_t __kprobes *get_insn_slot(void) { struct kprobe_insn_page *kip; struct hlist_node *pos; + kprobe_opcode_t *ret; + mutex_lock(&kprobe_insn_mutex); retry: hlist_for_each_entry(kip, pos, &kprobe_insn_pages, hlist) { if (kip->nused < INSNS_PER_PAGE) { @@ -152,7 +159,8 @@ kprobe_opcode_t __kprobes *get_insn_slot(void) if (kip->slot_used[i] == SLOT_CLEAN) { kip->slot_used[i] = SLOT_USED; kip->nused++; - return kip->insns + (i * MAX_INSN_SIZE); + ret = kip->insns + (i * MAX_INSN_SIZE); + goto end; } } /* Surprise! No unused slots. Fix kip->nused. */ @@ -166,8 +174,10 @@ kprobe_opcode_t __kprobes *get_insn_slot(void) } /* All out of space. Need to allocate a new page. Use slot 0. */ kip = kmalloc(sizeof(struct kprobe_insn_page), GFP_KERNEL); - if (!kip) - return NULL; + if (!kip) { + ret = NULL; + goto end; + } /* * Use module_alloc so this page is within +/- 2GB of where the @@ -177,7 +187,8 @@ kprobe_opcode_t __kprobes *get_insn_slot(void) kip->insns = module_alloc(PAGE_SIZE); if (!kip->insns) { kfree(kip); - return NULL; + ret = NULL; + goto end; } INIT_HLIST_NODE(&kip->hlist); hlist_add_head(&kip->hlist, &kprobe_insn_pages); @@ -185,7 +196,10 @@ kprobe_opcode_t __kprobes *get_insn_slot(void) kip->slot_used[0] = SLOT_USED; kip->nused = 1; kip->ngarbage = 0; - return kip->insns; + ret = kip->insns; +end: + mutex_unlock(&kprobe_insn_mutex); + return ret; } /* Return 1 if all garbages are collected, otherwise 0. */ @@ -219,7 +233,7 @@ static int __kprobes collect_garbage_slots(void) struct kprobe_insn_page *kip; struct hlist_node *pos, *next; - /* Ensure no-one is preepmted on the garbages */ + /* Ensure no-one is preempted on the garbages */ if (check_safety() != 0) return -EAGAIN; @@ -243,6 +257,7 @@ void __kprobes free_insn_slot(kprobe_opcode_t * slot, int dirty) struct kprobe_insn_page *kip; struct hlist_node *pos; + mutex_lock(&kprobe_insn_mutex); hlist_for_each_entry(kip, pos, &kprobe_insn_pages, hlist) { if (kip->insns <= slot && slot < kip->insns + (INSNS_PER_PAGE * MAX_INSN_SIZE)) { @@ -259,6 +274,7 @@ void __kprobes free_insn_slot(kprobe_opcode_t * slot, int dirty) if (dirty && ++kprobe_garbage_slots > INSNS_PER_PAGE) collect_garbage_slots(); + mutex_unlock(&kprobe_insn_mutex); } #endif @@ -603,9 +619,10 @@ static int __kprobes __register_kprobe(struct kprobe *p, goto out; } + kernel_text_lock(); ret = arch_prepare_kprobe(p); if (ret) - goto out; + goto out_unlock_text; INIT_HLIST_NODE(&p->hlist); hlist_add_head_rcu(&p->hlist, @@ -613,7 +630,8 @@ static int __kprobes __register_kprobe(struct kprobe *p, if (kprobe_enabled) arch_arm_kprobe(p); - +out_unlock_text: + kernel_text_unlock(); out: mutex_unlock(&kprobe_mutex); @@ -649,8 +667,11 @@ valid_p: * enabled - otherwise, the breakpoint would already have * been removed. We save on flushing icache. */ - if (kprobe_enabled) + if (kprobe_enabled) { + kernel_text_lock(); arch_disarm_kprobe(p); + kernel_text_unlock(); + } hlist_del_rcu(&old_p->hlist); } else { if (p->break_handler) @@ -847,7 +868,6 @@ static int __kprobes pre_handler_kretprobe(struct kprobe *p, } arch_prepare_kretprobe(ri, regs); - /* XXX(hch): why is there no hlist_move_head? */ hlist_del(&ri->uflist); hlist_add_head(&ri->uflist, &ri->rp->used_instances); @@ -1147,11 +1167,13 @@ static void __kprobes enable_all_kprobes(void) if (kprobe_enabled) goto already_enabled; + kernel_text_lock(); for (i = 0; i < KPROBE_TABLE_SIZE; i++) { head = &kprobe_table[i]; hlist_for_each_entry_rcu(p, node, head, hlist) arch_arm_kprobe(p); } + kernel_text_unlock(); kprobe_enabled = true; printk(KERN_INFO "Kprobes globally enabled\n"); @@ -1176,6 +1198,7 @@ static void __kprobes disable_all_kprobes(void) kprobe_enabled = false; printk(KERN_INFO "Kprobes globally disabled\n"); + kernel_text_lock(); for (i = 0; i < KPROBE_TABLE_SIZE; i++) { head = &kprobe_table[i]; hlist_for_each_entry_rcu(p, node, head, hlist) { @@ -1183,6 +1206,7 @@ static void __kprobes disable_all_kprobes(void) arch_disarm_kprobe(p); } } + kernel_text_unlock(); mutex_unlock(&kprobe_mutex); /* Allow all currently running kprobes to complete */ diff --git a/kernel/kthread.c b/kernel/kthread.c index 9eb21645765f..9b098ff2c847 100644 --- a/kernel/kthread.c +++ b/kernel/kthread.c @@ -13,6 +13,9 @@ #include <linux/file.h> #include <linux/module.h> #include <linux/mutex.h> +#include <linux/cpumask.h> +#include <linux/cpuset.h> +#include <linux/marker.h> #define KTHREAD_NICE_LEVEL (-5) @@ -106,7 +109,7 @@ static void create_kthread(struct kthread_create_info *create) */ sched_setscheduler(create->result, SCHED_NORMAL, ¶m); set_user_nice(create->result, KTHREAD_NICE_LEVEL); - set_cpus_allowed(create->result, CPU_MASK_ALL); + set_cpus_allowed_ptr(create->result, &cpu_system_map); } complete(&create->done); } @@ -186,6 +189,8 @@ int kthread_stop(struct task_struct *k) /* It could exit after stop_info.k set, but before wake_up_process. */ get_task_struct(k); + trace_mark(kernel_kthread_stop, "pid %d", k->pid); + /* Must init completion *before* thread sees kthread_stop_info.k */ init_completion(&kthread_stop_info.done); smp_wmb(); @@ -201,6 +206,8 @@ int kthread_stop(struct task_struct *k) ret = kthread_stop_info.err; mutex_unlock(&kthread_stop_lock); + trace_mark(kernel_kthread_stop_ret, "ret %d", ret); + return ret; } EXPORT_SYMBOL(kthread_stop); @@ -213,7 +220,7 @@ int kthreadd(void *unused) set_task_comm(tsk, "kthreadd"); ignore_signals(tsk); set_user_nice(tsk, KTHREAD_NICE_LEVEL); - set_cpus_allowed(tsk, CPU_MASK_ALL); + set_cpus_allowed_ptr(tsk, &cpu_system_map); current->flags |= PF_NOFREEZE; @@ -241,3 +248,47 @@ int kthreadd(void *unused) return 0; } + +#ifdef CONFIG_CPUSETS +static int system_kthread_notifier(struct notifier_block *nb, + unsigned long action, void *cpus) +{ + cpumask_t *new_system_map = (cpumask_t *)cpus; + struct task_struct *g, *t; + +again: + rcu_read_lock(); + do_each_thread(g, t) { + if (t->parent != kthreadd_task && t != kthreadd_task) + continue; + + if (cpus_match_system(t->cpus_allowed) && + !cpus_equal(t->cpus_allowed, *new_system_map)) { + /* + * What is holding a ref on t->usage here?! + */ + get_task_struct(t); + rcu_read_unlock(); + set_cpus_allowed_ptr(t, new_system_map); + put_task_struct(t); + goto again; + } + } while_each_thread(g, t); + rcu_read_unlock(); + + return NOTIFY_OK; +} + +static struct notifier_block fn_system_kthread_notifier = { + .notifier_call = system_kthread_notifier, +}; + +static int __init init_kthread(void) +{ + blocking_notifier_chain_register(&system_map_notifier, + &fn_system_kthread_notifier); + return 0; +} + +module_init(init_kthread); +#endif diff --git a/kernel/lockdep.c b/kernel/lockdep.c index 65548eff029e..23230a081bb3 100644 --- a/kernel/lockdep.c +++ b/kernel/lockdep.c @@ -40,6 +40,7 @@ #include <linux/utsname.h> #include <linux/hash.h> #include <linux/ftrace.h> +#include <linux/marker.h> #include <asm/sections.h> @@ -2024,6 +2025,9 @@ void trace_hardirqs_on_caller(unsigned long a0) time_hardirqs_on(CALLER_ADDR0, a0); + _trace_mark(locking_hardirqs_on, "ip #p%lu", + (unsigned long) __builtin_return_address(0)); + if (unlikely(!debug_locks || current->lockdep_recursion)) return; @@ -2078,6 +2082,9 @@ void trace_hardirqs_off_caller(unsigned long a0) time_hardirqs_off(CALLER_ADDR0, a0); + _trace_mark(locking_hardirqs_off, "ip #p%lu", + (unsigned long) __builtin_return_address(0)); + if (unlikely(!debug_locks || current->lockdep_recursion)) return; @@ -2110,6 +2117,9 @@ void trace_softirqs_on(unsigned long ip) { struct task_struct *curr = current; + _trace_mark(locking_softirqs_on, "ip #p%lu", + (unsigned long) __builtin_return_address(0)); + if (unlikely(!debug_locks)) return; @@ -2144,6 +2154,9 @@ void trace_softirqs_off(unsigned long ip) { struct task_struct *curr = current; + _trace_mark(locking_softirqs_off, "ip #p%lu", + (unsigned long) __builtin_return_address(0)); + if (unlikely(!debug_locks)) return; @@ -2376,6 +2389,10 @@ static int __lock_acquire(struct lockdep_map *lock, unsigned int subclass, int chain_head = 0; u64 chain_key; + _trace_mark(locking_lock_acquire, + "ip #p%lu subclass %u lock %p trylock %d", + ip, subclass, lock, trylock); + if (!prove_locking) check = 1; @@ -2649,6 +2666,9 @@ __lock_release(struct lockdep_map *lock, int nested, unsigned long ip) { struct task_struct *curr = current; + _trace_mark(locking_lock_release, "ip #p%lu lock %p nested %d", + ip, lock, nested); + if (!check_unlock(curr, lock, ip)) return; diff --git a/kernel/marker.c b/kernel/marker.c index b5a9fe1d50d5..f6fc0bcd6034 100644 --- a/kernel/marker.c +++ b/kernel/marker.c @@ -24,6 +24,7 @@ #include <linux/marker.h> #include <linux/err.h> #include <linux/slab.h> +#include <linux/immediate.h> extern struct marker __start___markers[]; extern struct marker __stop___markers[]; @@ -55,8 +56,8 @@ static DEFINE_MUTEX(markers_mutex); struct marker_entry { struct hlist_node hlist; char *format; - void (*call)(const struct marker *mdata, /* Probe wrapper */ - void *call_private, const char *fmt, ...); + /* Probe wrapper */ + void (*call)(const struct marker *mdata, void *call_private, ...); struct marker_probe_closure single; struct marker_probe_closure *multi; int refcount; /* Number of times armed. 0 if disarmed. */ @@ -91,15 +92,13 @@ EXPORT_SYMBOL_GPL(__mark_empty_function); * marker_probe_cb Callback that prepares the variable argument list for probes. * @mdata: pointer of type struct marker * @call_private: caller site private data - * @fmt: format string * @...: Variable argument list. * * Since we do not use "typical" pointer based RCU in the 1 argument case, we * need to put a full smp_rmb() in this branch. This is why we do not use * rcu_dereference() for the pointer read. */ -void marker_probe_cb(const struct marker *mdata, void *call_private, - const char *fmt, ...) +void marker_probe_cb(const struct marker *mdata, void *call_private, ...) { va_list args; char ptype; @@ -120,8 +119,9 @@ void marker_probe_cb(const struct marker *mdata, void *call_private, /* Must read the ptr before private data. They are not data * dependant, so we put an explicit smp_rmb() here. */ smp_rmb(); - va_start(args, fmt); - func(mdata->single.probe_private, call_private, fmt, &args); + va_start(args, call_private); + func(mdata->single.probe_private, call_private, mdata->format, + &args); va_end(args); } else { struct marker_probe_closure *multi; @@ -136,9 +136,9 @@ void marker_probe_cb(const struct marker *mdata, void *call_private, smp_read_barrier_depends(); multi = mdata->multi; for (i = 0; multi[i].func; i++) { - va_start(args, fmt); - multi[i].func(multi[i].probe_private, call_private, fmt, - &args); + va_start(args, call_private); + multi[i].func(multi[i].probe_private, call_private, + mdata->format, &args); va_end(args); } } @@ -150,13 +150,11 @@ EXPORT_SYMBOL_GPL(marker_probe_cb); * marker_probe_cb Callback that does not prepare the variable argument list. * @mdata: pointer of type struct marker * @call_private: caller site private data - * @fmt: format string * @...: Variable argument list. * * Should be connected to markers "MARK_NOARGS". */ -void marker_probe_cb_noarg(const struct marker *mdata, - void *call_private, const char *fmt, ...) +void marker_probe_cb_noarg(const struct marker *mdata, void *call_private, ...) { va_list args; /* not initialized */ char ptype; @@ -172,7 +170,8 @@ void marker_probe_cb_noarg(const struct marker *mdata, /* Must read the ptr before private data. They are not data * dependant, so we put an explicit smp_rmb() here. */ smp_rmb(); - func(mdata->single.probe_private, call_private, fmt, &args); + func(mdata->single.probe_private, call_private, mdata->format, + &args); } else { struct marker_probe_closure *multi; int i; @@ -186,8 +185,8 @@ void marker_probe_cb_noarg(const struct marker *mdata, smp_read_barrier_depends(); multi = mdata->multi; for (i = 0; multi[i].func; i++) - multi[i].func(multi[i].probe_private, call_private, fmt, - &args); + multi[i].func(multi[i].probe_private, call_private, + mdata->format, &args); } preempt_enable(); } @@ -545,7 +544,7 @@ static int set_marker(struct marker_entry **entry, struct marker *elem, */ smp_wmb(); elem->ptype = (*entry)->ptype; - elem->state = active; + elem->state__imv = active; return 0; } @@ -559,7 +558,7 @@ static int set_marker(struct marker_entry **entry, struct marker *elem, static void disable_marker(struct marker *elem) { /* leave "call" as is. It is known statically. */ - elem->state = 0; + elem->state__imv = 0; elem->single.func = __mark_empty_function; /* Update the function before setting the ptype */ smp_wmb(); @@ -623,6 +622,9 @@ static void marker_update_probes(void) marker_update_probe_range(__start___markers, __stop___markers); /* Markers in modules. */ module_update_markers(); + /* Update immediate values */ + core_imv_update(); + module_imv_update(); } /** diff --git a/kernel/module.c b/kernel/module.c index 8e4528c9909f..39fa1837579c 100644 --- a/kernel/module.c +++ b/kernel/module.c @@ -33,6 +33,7 @@ #include <linux/cpu.h> #include <linux/moduleparam.h> #include <linux/errno.h> +#include <linux/immediate.h> #include <linux/err.h> #include <linux/vermagic.h> #include <linux/notifier.h> @@ -46,6 +47,7 @@ #include <asm/cacheflush.h> #include <linux/license.h> #include <asm/sections.h> +#include <linux/marker.h> #if 0 #define DEBUGP printk @@ -1364,6 +1366,8 @@ static int __unlink_module(void *_mod) /* Free a module, remove from lists, etc (must hold module_mutex). */ static void free_module(struct module *mod) { + trace_mark(kernel_module_free, "name %s", mod->name); + /* Delete from various lists */ stop_machine_run(__unlink_module, mod, NR_CPUS); remove_notes_attrs(mod); @@ -1747,6 +1751,8 @@ static struct module *load_module(void __user *umod, unsigned int unusedcrcindex; unsigned int unusedgplindex; unsigned int unusedgplcrcindex; + unsigned int immediateindex; + unsigned int immediatecondendindex; unsigned int markersindex; unsigned int markersstringsindex; struct module *mod; @@ -1845,6 +1851,9 @@ static struct module *load_module(void __user *umod, #ifdef ARCH_UNWIND_SECTION_NAME unwindex = find_sec(hdr, sechdrs, secstrings, ARCH_UNWIND_SECTION_NAME); #endif + immediateindex = find_sec(hdr, sechdrs, secstrings, "__imv"); + immediatecondendindex = find_sec(hdr, sechdrs, secstrings, + "__imv_cond_end"); /* Don't keep modinfo and version sections. */ sechdrs[infoindex].sh_flags &= ~(unsigned long)SHF_ALLOC; @@ -2004,6 +2013,16 @@ static struct module *load_module(void __user *umod, mod->gpl_future_syms = (void *)sechdrs[gplfutureindex].sh_addr; if (gplfuturecrcindex) mod->gpl_future_crcs = (void *)sechdrs[gplfuturecrcindex].sh_addr; +#ifdef CONFIG_IMMEDIATE + mod->immediate = (void *)sechdrs[immediateindex].sh_addr; + mod->num_immediate = + sechdrs[immediateindex].sh_size / sizeof(*mod->immediate); + mod->immediate_cond_end = + (void *)sechdrs[immediatecondendindex].sh_addr; + mod->num_immediate_cond_end = + sechdrs[immediatecondendindex].sh_size + / sizeof(*mod->immediate_cond_end); +#endif mod->unused_syms = (void *)sechdrs[unusedindex].sh_addr; if (unusedcrcindex) @@ -2073,11 +2092,17 @@ static struct module *load_module(void __user *umod, add_kallsyms(mod, sechdrs, symindex, strindex, secstrings); + if (!(mod->taints & TAINT_FORCED_MODULE)) { #ifdef CONFIG_MARKERS - if (!mod->taints) marker_update_probe_range(mod->markers, mod->markers + mod->num_markers); #endif +#ifdef CONFIG_IMMEDIATE + /* Immediate values must be updated after markers */ + imv_update_range(mod->immediate, + mod->immediate + mod->num_immediate); +#endif +} err = module_finalize(hdr, sechdrs, mod); if (err < 0) goto cleanup; @@ -2138,6 +2163,8 @@ static struct module *load_module(void __user *umod, /* Get rid of temporary copy */ vfree(hdr); + trace_mark(kernel_module_load, "name %s", mod->name); + /* Done! */ return mod; @@ -2231,6 +2258,10 @@ sys_init_module(void __user *umod, /* Drop initial reference. */ module_put(mod); unwind_remove_table(mod->unwind_info, 1); +#ifdef CONFIG_IMMEDIATE + imv_unref(mod->immediate, mod->immediate + mod->num_immediate, + mod->module_init, mod->init_size); +#endif module_free(mod, mod->module_init); mod->module_init = NULL; mod->init_size = 0; @@ -2625,3 +2656,53 @@ void module_update_markers(void) mutex_unlock(&module_mutex); } #endif + +#ifdef CONFIG_IMMEDIATE +/** + * _module_imv_update - update all immediate values in the kernel + * + * Iterate on the kernel core and modules to update the immediate values. + * Module_mutex must be held be the caller. + */ +void _module_imv_update(void) +{ + struct module *mod; + + list_for_each_entry(mod, &modules, list) { + if (mod->taints) + continue; + imv_update_range(mod->immediate, + mod->immediate + mod->num_immediate); + } +} +EXPORT_SYMBOL_GPL(_module_imv_update); + +/** + * module_imv_update - update all immediate values in the kernel + * + * Iterate on the kernel core and modules to update the immediate values. + * Takes module_mutex. + */ +void module_imv_update(void) +{ + mutex_lock(&module_mutex); + _module_imv_update(); + mutex_unlock(&module_mutex); +} +EXPORT_SYMBOL_GPL(module_imv_update); + +/** + * is_imv_cond_end_module + * + * Check if the two given addresses are located in the immediate value condition + * end table. Addresses should be in the same object. + * The module mutex should be held. + */ +int is_imv_cond_end_module(unsigned long addr1, unsigned long addr2) +{ + struct module *mod = __module_text_address(addr1); + return _is_imv_cond_end(mod->immediate_cond_end, + mod->immediate_cond_end + mod->num_immediate_cond_end, + addr1, addr2); +} +#endif diff --git a/kernel/pid.c b/kernel/pid.c index 20d59fa2d493..30bd5d4b2ac7 100644 --- a/kernel/pid.c +++ b/kernel/pid.c @@ -30,6 +30,7 @@ #include <linux/module.h> #include <linux/slab.h> #include <linux/init.h> +#include <linux/rculist.h> #include <linux/bootmem.h> #include <linux/hash.h> #include <linux/pid_namespace.h> diff --git a/kernel/printk.c b/kernel/printk.c index 6c58b83fde1e..e585d125932f 100644 --- a/kernel/printk.c +++ b/kernel/printk.c @@ -32,6 +32,7 @@ #include <linux/security.h> #include <linux/bootmem.h> #include <linux/syscalls.h> +#include <linux/marker.h> #include <asm/uaccess.h> @@ -231,7 +232,7 @@ static inline void boot_delay_msec(void) /* * Return the number of unread characters in the log buffer. */ -int log_buf_get_len(void) +static int log_buf_get_len(void) { return logged_chars; } @@ -268,19 +269,6 @@ int log_buf_copy(char *dest, int idx, int len) } /* - * Extract a single character from the log buffer. - */ -int log_buf_read(int idx) -{ - char ret; - - if (log_buf_copy(&ret, idx, 1) == 1) - return ret; - else - return -1; -} - -/* * Commands to do_syslog: * * 0 -- Close the log. Currently a NOP. @@ -610,6 +598,7 @@ asmlinkage int printk(const char *fmt, ...) int r; va_start(args, fmt); + trace_mark(kernel_printk, "ip %p", __builtin_return_address(0)); r = vprintk(fmt, args); va_end(args); @@ -665,18 +654,17 @@ static int acquire_console_semaphore_for_printk(unsigned int cpu) spin_unlock(&logbuf_lock); return retval; } - -const char printk_recursion_bug_msg [] = - KERN_CRIT "BUG: recent printk recursion!\n"; -static int printk_recursion_bug; +static const char recursion_bug_msg [] = + KERN_CRIT "BUG: recent printk recursion!\n"; +static int recursion_bug; + static int new_text_line = 1; +static char printk_buf[1024]; asmlinkage int vprintk(const char *fmt, va_list args) { - static int log_level_unknown = 1; - static char printk_buf[1024]; - - unsigned long flags; int printed_len = 0; + int current_log_level = default_message_loglevel; + unsigned long flags; int this_cpu; char *p; @@ -687,6 +675,31 @@ asmlinkage int vprintk(const char *fmt, va_list args) raw_local_irq_save(flags); this_cpu = smp_processor_id(); + if (printed_len > 0) { + unsigned int loglevel; + int mark_len; + char *mark_buf; + char saved_char; + + if (printk_buf[0] == '<' && printk_buf[1] >= '0' && + printk_buf[1] <= '7' && printk_buf[2] == '>') { + loglevel = printk_buf[1] - '0'; + mark_buf = &printk_buf[3]; + mark_len = printed_len - 3; + } else { + loglevel = default_message_loglevel; + mark_buf = printk_buf; + mark_len = printed_len; + } + if (mark_buf[mark_len - 1] == '\n') + mark_len--; + saved_char = mark_buf[mark_len]; + mark_buf[mark_len] = '\0'; + _trace_mark(kernel_vprintk, "loglevel %c string %s ip %p", + loglevel, mark_buf, __builtin_return_address(0)); + mark_buf[mark_len] = saved_char; + } + /* * Ouch, printk recursed into itself! */ @@ -699,7 +712,7 @@ asmlinkage int vprintk(const char *fmt, va_list args) * it can be printed at the next appropriate moment: */ if (!oops_in_progress) { - printk_recursion_bug = 1; + recursion_bug = 1; goto out_restore_irqs; } zap_locks(); @@ -709,70 +722,62 @@ asmlinkage int vprintk(const char *fmt, va_list args) spin_lock(&logbuf_lock); printk_cpu = this_cpu; - if (printk_recursion_bug) { - printk_recursion_bug = 0; - strcpy(printk_buf, printk_recursion_bug_msg); - printed_len = sizeof(printk_recursion_bug_msg); + if (recursion_bug) { + recursion_bug = 0; + strcpy(printk_buf, recursion_bug_msg); + printed_len = sizeof(recursion_bug_msg); } /* Emit the output into the temporary buffer */ printed_len += vscnprintf(printk_buf + printed_len, sizeof(printk_buf) - printed_len, fmt, args); + /* * Copy the output into log_buf. If the caller didn't provide * appropriate log level tags, we insert them here */ for (p = printk_buf; *p; p++) { - if (log_level_unknown) { - /* log_level_unknown signals the start of a new line */ + if (new_text_line) { + /* If a token, set current_log_level and skip over */ + if (p[0] == '<' && p[1] >= '0' && p[1] <= '7' && + p[2] == '>') { + current_log_level = p[1] - '0'; + p += 3; + printed_len -= 3; + } + + /* Always output the token */ + emit_log_char('<'); + emit_log_char(current_log_level + '0'); + emit_log_char('>'); + printed_len += 3; + new_text_line = 0; + if (printk_time) { - int loglev_char; + /* Follow the token with the time */ char tbuf[50], *tp; unsigned tlen; unsigned long long t; unsigned long nanosec_rem; - /* - * force the log level token to be - * before the time output. - */ - if (p[0] == '<' && p[1] >='0' && - p[1] <= '7' && p[2] == '>') { - loglev_char = p[1]; - p += 3; - printed_len -= 3; - } else { - loglev_char = default_message_loglevel - + '0'; - } t = cpu_clock(printk_cpu); nanosec_rem = do_div(t, 1000000000); - tlen = sprintf(tbuf, - "<%c>[%5lu.%06lu] ", - loglev_char, - (unsigned long)t, - nanosec_rem/1000); + tlen = sprintf(tbuf, "[%5lu.%06lu] ", + (unsigned long) t, + nanosec_rem / 1000); for (tp = tbuf; tp < tbuf + tlen; tp++) emit_log_char(*tp); printed_len += tlen; - } else { - if (p[0] != '<' || p[1] < '0' || - p[1] > '7' || p[2] != '>') { - emit_log_char('<'); - emit_log_char(default_message_loglevel - + '0'); - emit_log_char('>'); - printed_len += 3; - } } - log_level_unknown = 0; + if (!*p) break; } + emit_log_char(*p); if (*p == '\n') - log_level_unknown = 1; + new_text_line = 1; } /* @@ -1174,8 +1179,11 @@ void register_console(struct console *console) console->index = 0; if (console->setup == NULL || console->setup(console, NULL) == 0) { - console->flags |= CON_ENABLED | CON_CONSDEV; - preferred_console = 0; + console->flags |= CON_ENABLED; + if (console->device) { + console->flags |= CON_CONSDEV; + preferred_console = 0; + } } } diff --git a/kernel/profile.c b/kernel/profile.c index ae7ead82cbc9..9b0e6c2fcbc6 100644 --- a/kernel/profile.c +++ b/kernel/profile.c @@ -40,8 +40,8 @@ static int (*timer_hook)(struct pt_regs *) __read_mostly; static atomic_t *prof_buffer; static unsigned long prof_len, prof_shift; -int prof_on __read_mostly; -EXPORT_SYMBOL_GPL(prof_on); +DEFINE_IMV(char, prof_on) __read_mostly; +EXPORT_IMV_SYMBOL_GPL(prof_on); static cpumask_t prof_cpu_mask = CPU_MASK_ALL; #ifdef CONFIG_SMP @@ -59,7 +59,7 @@ static int __init profile_setup(char *str) if (!strncmp(str, sleepstr, strlen(sleepstr))) { #ifdef CONFIG_SCHEDSTATS - prof_on = SLEEP_PROFILING; + imv_set(prof_on, SLEEP_PROFILING); if (str[strlen(sleepstr)] == ',') str += strlen(sleepstr) + 1; if (get_option(&str, &par)) @@ -72,7 +72,7 @@ static int __init profile_setup(char *str) "kernel sleep profiling requires CONFIG_SCHEDSTATS\n"); #endif /* CONFIG_SCHEDSTATS */ } else if (!strncmp(str, schedstr, strlen(schedstr))) { - prof_on = SCHED_PROFILING; + imv_set(prof_on, SCHED_PROFILING); if (str[strlen(schedstr)] == ',') str += strlen(schedstr) + 1; if (get_option(&str, &par)) @@ -81,7 +81,7 @@ static int __init profile_setup(char *str) "kernel schedule profiling enabled (shift: %ld)\n", prof_shift); } else if (!strncmp(str, kvmstr, strlen(kvmstr))) { - prof_on = KVM_PROFILING; + imv_set(prof_on, KVM_PROFILING); if (str[strlen(kvmstr)] == ',') str += strlen(kvmstr) + 1; if (get_option(&str, &par)) @@ -91,7 +91,7 @@ static int __init profile_setup(char *str) prof_shift); } else if (get_option(&str, &par)) { prof_shift = par; - prof_on = CPU_PROFILING; + imv_set(prof_on, CPU_PROFILING); printk(KERN_INFO "kernel profiling enabled (shift: %ld)\n", prof_shift); } @@ -102,7 +102,7 @@ __setup("profile=", profile_setup); void __init profile_init(void) { - if (!prof_on) + if (!_imv_read(prof_on)) return; /* only text is profiled */ @@ -289,7 +289,7 @@ void profile_hits(int type, void *__pc, unsigned int nr_hits) int i, j, cpu; struct profile_hit *hits; - if (prof_on != type || !prof_buffer) + if (!prof_buffer) return; pc = min((pc - (unsigned long)_stext) >> prof_shift, prof_len - 1); i = primary = (pc & (NR_PROFILE_GRP - 1)) << PROFILE_GRPSHIFT; @@ -399,7 +399,7 @@ void profile_hits(int type, void *__pc, unsigned int nr_hits) { unsigned long pc; - if (prof_on != type || !prof_buffer) + if (!prof_buffer) return; pc = ((unsigned long)__pc - (unsigned long)_stext) >> prof_shift; atomic_add(nr_hits, &prof_buffer[min(pc, prof_len - 1)]); @@ -556,7 +556,7 @@ static int __init create_hash_tables(void) } return 0; out_cleanup: - prof_on = 0; + imv_set(prof_on, 0); smp_mb(); on_each_cpu(profile_nop, NULL, 0, 1); for_each_online_cpu(cpu) { @@ -583,7 +583,7 @@ static int __init create_proc_profile(void) { struct proc_dir_entry *entry; - if (!prof_on) + if (!_imv_read(prof_on)) return 0; if (create_hash_tables()) return -1; diff --git a/kernel/rcuclassic.c b/kernel/rcuclassic.c index f4ffbd0f306f..02c75a2ff70e 100644 --- a/kernel/rcuclassic.c +++ b/kernel/rcuclassic.c @@ -92,7 +92,7 @@ static void force_quiescent_state(struct rcu_data *rdp, */ cpumask = rcp->cpumask; cpu_clear(rdp->cpu, cpumask); - for_each_cpu_mask(cpu, cpumask) + for_each_cpu_mask_nr(cpu, cpumask) smp_send_reschedule(cpu); } } @@ -502,10 +502,38 @@ void rcu_check_callbacks(int cpu, int user) if (user || (idle_cpu(cpu) && !in_softirq() && hardirq_count() <= (1 << HARDIRQ_SHIFT))) { + + /* + * Get here if this CPU took its interrupt from user + * mode or from the idle loop, and if this is not a + * nested interrupt. In this case, the CPU is in + * a quiescent state, so count it. + * + * Also do a memory barrier. This is needed to handle + * the case where writes from a preempt-disable section + * of code get reordered into schedule() by this CPU's + * write buffer. The memory barrier makes sure that + * the rcu_qsctr_inc() and rcu_bh_qsctr_inc() are see + * by other CPUs to happen after any such write. + */ + + smp_mb(); /* See above block comment. */ rcu_qsctr_inc(cpu); rcu_bh_qsctr_inc(cpu); - } else if (!in_softirq()) + + } else if (!in_softirq()) { + + /* + * Get here if this CPU did not take its interrupt from + * softirq, in other words, if it is not interrupting + * a rcu_bh read-side critical section. This is an _bh + * critical section, so count it. The memory barrier + * is needed for the same reason as is the above one. + */ + + smp_mb(); /* See above block comment. */ rcu_bh_qsctr_inc(cpu); + } raise_rcu_softirq(); } diff --git a/kernel/rcupdate.c b/kernel/rcupdate.c index c09605f8d16c..4a74b8d48d90 100644 --- a/kernel/rcupdate.c +++ b/kernel/rcupdate.c @@ -39,16 +39,16 @@ #include <linux/sched.h> #include <asm/atomic.h> #include <linux/bitops.h> -#include <linux/completion.h> #include <linux/percpu.h> #include <linux/notifier.h> #include <linux/cpu.h> #include <linux/mutex.h> #include <linux/module.h> -struct rcu_synchronize { - struct rcu_head head; - struct completion completion; +enum rcu_barrier { + RCU_BARRIER_STD, + RCU_BARRIER_BH, + RCU_BARRIER_SCHED, }; static DEFINE_PER_CPU(struct rcu_head, rcu_barrier_head) = {NULL}; @@ -60,7 +60,7 @@ static struct completion rcu_barrier_completion; * Awaken the corresponding synchronize_rcu() instance now that a * grace period has elapsed. */ -static void wakeme_after_rcu(struct rcu_head *head) +void wakeme_after_rcu(struct rcu_head *head) { struct rcu_synchronize *rcu; @@ -77,17 +77,7 @@ static void wakeme_after_rcu(struct rcu_head *head) * sections are delimited by rcu_read_lock() and rcu_read_unlock(), * and may be nested. */ -void synchronize_rcu(void) -{ - struct rcu_synchronize rcu; - - init_completion(&rcu.completion); - /* Will wake me after RCU finished */ - call_rcu(&rcu.head, wakeme_after_rcu); - - /* Wait for it */ - wait_for_completion(&rcu.completion); -} +synchronize_rcu_xxx(synchronize_rcu, call_rcu) EXPORT_SYMBOL_GPL(synchronize_rcu); static void rcu_barrier_callback(struct rcu_head *notused) @@ -99,19 +89,30 @@ static void rcu_barrier_callback(struct rcu_head *notused) /* * Called with preemption disabled, and from cross-cpu IRQ context. */ -static void rcu_barrier_func(void *notused) +static void rcu_barrier_func(void *type) { int cpu = smp_processor_id(); struct rcu_head *head = &per_cpu(rcu_barrier_head, cpu); atomic_inc(&rcu_barrier_cpu_count); - call_rcu(head, rcu_barrier_callback); + switch ((enum rcu_barrier)type) { + case RCU_BARRIER_STD: + call_rcu(head, rcu_barrier_callback); + break; + case RCU_BARRIER_BH: + call_rcu_bh(head, rcu_barrier_callback); + break; + case RCU_BARRIER_SCHED: + call_rcu_sched(head, rcu_barrier_callback); + break; + } } -/** - * rcu_barrier - Wait until all the in-flight RCUs are complete. +/* + * Orchestrate the specified type of RCU barrier, waiting for all + * RCU callbacks of the specified type to complete. */ -void rcu_barrier(void) +static void _rcu_barrier(enum rcu_barrier type) { BUG_ON(in_interrupt()); /* Take cpucontrol mutex to protect against CPU hotplug */ @@ -127,13 +128,39 @@ void rcu_barrier(void) * until all the callbacks are queued. */ rcu_read_lock(); - on_each_cpu(rcu_barrier_func, NULL, 0, 1); + on_each_cpu(rcu_barrier_func, (void *)type, 0, 1); rcu_read_unlock(); wait_for_completion(&rcu_barrier_completion); mutex_unlock(&rcu_barrier_mutex); } + +/** + * rcu_barrier - Wait until all in-flight call_rcu() callbacks complete. + */ +void rcu_barrier(void) +{ + _rcu_barrier(RCU_BARRIER_STD); +} EXPORT_SYMBOL_GPL(rcu_barrier); +/** + * rcu_barrier_bh - Wait until all in-flight call_rcu_bh() callbacks complete. + */ +void rcu_barrier_bh(void) +{ + _rcu_barrier(RCU_BARRIER_BH); +} +EXPORT_SYMBOL_GPL(rcu_barrier_bh); + +/** + * rcu_barrier_sched - Wait for in-flight call_rcu_sched() callbacks. + */ +void rcu_barrier_sched(void) +{ + _rcu_barrier(RCU_BARRIER_SCHED); +} +EXPORT_SYMBOL_GPL(rcu_barrier_sched); + void __init rcu_init(void) { __rcu_init(); diff --git a/kernel/rcupreempt.c b/kernel/rcupreempt.c index e1cdf196a515..80d42e80e963 100644 --- a/kernel/rcupreempt.c +++ b/kernel/rcupreempt.c @@ -46,6 +46,7 @@ #include <asm/atomic.h> #include <linux/bitops.h> #include <linux/module.h> +#include <linux/kthread.h> #include <linux/completion.h> #include <linux/moduleparam.h> #include <linux/percpu.h> @@ -87,9 +88,14 @@ struct rcu_data { struct rcu_head **nexttail; struct rcu_head *waitlist[GP_STAGES]; struct rcu_head **waittail[GP_STAGES]; - struct rcu_head *donelist; + struct rcu_head *donelist; /* from waitlist & waitschedlist */ struct rcu_head **donetail; long rcu_flipctr[2]; + struct rcu_head *nextschedlist; + struct rcu_head **nextschedtail; + struct rcu_head *waitschedlist; + struct rcu_head **waitschedtail; + int rcu_sched_sleeping; #ifdef CONFIG_RCU_TRACE struct rcupreempt_trace trace; #endif /* #ifdef CONFIG_RCU_TRACE */ @@ -131,11 +137,24 @@ enum rcu_try_flip_states { rcu_try_flip_waitmb_state, }; +/* + * States for rcu_ctrlblk.rcu_sched_sleep. + */ + +enum rcu_sched_sleep_states { + rcu_sched_not_sleeping, /* Not sleeping, callbacks need GP. */ + rcu_sched_sleep_prep, /* Thinking of sleeping, rechecking. */ + rcu_sched_sleeping, /* Sleeping, awaken if GP needed. */ +}; + struct rcu_ctrlblk { spinlock_t fliplock; /* Protect state-machine transitions. */ long completed; /* Number of last completed batch. */ enum rcu_try_flip_states rcu_try_flip_state; /* The current state of the rcu state machine */ + spinlock_t schedlock; /* Protect rcu_sched sleep state. */ + enum rcu_sched_sleep_states sched_sleep; /* rcu_sched state. */ + wait_queue_head_t sched_wq; /* Place for rcu_sched to sleep. */ }; static DEFINE_PER_CPU(struct rcu_data, rcu_data); @@ -143,8 +162,12 @@ static struct rcu_ctrlblk rcu_ctrlblk = { .fliplock = __SPIN_LOCK_UNLOCKED(rcu_ctrlblk.fliplock), .completed = 0, .rcu_try_flip_state = rcu_try_flip_idle_state, + .schedlock = __SPIN_LOCK_UNLOCKED(rcu_ctrlblk.schedlock), + .sched_sleep = rcu_sched_not_sleeping, + .sched_wq = __WAIT_QUEUE_HEAD_INITIALIZER(rcu_ctrlblk.sched_wq), }; +static struct task_struct *rcu_sched_grace_period_task; #ifdef CONFIG_RCU_TRACE static char *rcu_try_flip_state_names[] = @@ -207,6 +230,8 @@ static DEFINE_PER_CPU_SHARED_ALIGNED(enum rcu_mb_flag_values, rcu_mb_flag) */ #define RCU_TRACE_RDP(f, rdp) RCU_TRACE(f, &((rdp)->trace)); +#define RCU_SCHED_BATCH_TIME (HZ / 50) + /* * Return the number of RCU batches processed thus far. Useful * for debug and statistics. @@ -217,8 +242,6 @@ long rcu_batches_completed(void) } EXPORT_SYMBOL_GPL(rcu_batches_completed); -EXPORT_SYMBOL_GPL(rcu_batches_completed_bh); - void __rcu_read_lock(void) { int idx; @@ -413,32 +436,34 @@ static void __rcu_advance_callbacks(struct rcu_data *rdp) } } -#ifdef CONFIG_NO_HZ +DEFINE_PER_CPU_SHARED_ALIGNED(struct rcu_dyntick_sched, rcu_dyntick_sched) = { + .dynticks = 1, +}; -DEFINE_PER_CPU(long, dynticks_progress_counter) = 1; -static DEFINE_PER_CPU(long, rcu_dyntick_snapshot); +#ifdef CONFIG_NO_HZ static DEFINE_PER_CPU(int, rcu_update_flag); /** * rcu_irq_enter - Called from Hard irq handlers and NMI/SMI. * * If the CPU was idle with dynamic ticks active, this updates the - * dynticks_progress_counter to let the RCU handling know that the + * rcu_dyntick_sched.dynticks to let the RCU handling know that the * CPU is active. */ void rcu_irq_enter(void) { int cpu = smp_processor_id(); + struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu); if (per_cpu(rcu_update_flag, cpu)) per_cpu(rcu_update_flag, cpu)++; /* * Only update if we are coming from a stopped ticks mode - * (dynticks_progress_counter is even). + * (rcu_dyntick_sched.dynticks is even). */ if (!in_interrupt() && - (per_cpu(dynticks_progress_counter, cpu) & 0x1) == 0) { + (rdssp->dynticks & 0x1) == 0) { /* * The following might seem like we could have a race * with NMI/SMIs. But this really isn't a problem. @@ -461,12 +486,12 @@ void rcu_irq_enter(void) * RCU read-side critical sections on this CPU would * have already completed. */ - per_cpu(dynticks_progress_counter, cpu)++; + rdssp->dynticks++; /* * The following memory barrier ensures that any * rcu_read_lock() primitives in the irq handler * are seen by other CPUs to follow the above - * increment to dynticks_progress_counter. This is + * increment to rcu_dyntick_sched.dynticks. This is * required in order for other CPUs to correctly * determine when it is safe to advance the RCU * grace-period state machine. @@ -474,7 +499,7 @@ void rcu_irq_enter(void) smp_mb(); /* see above block comment. */ /* * Since we can't determine the dynamic tick mode from - * the dynticks_progress_counter after this routine, + * the rcu_dyntick_sched.dynticks after this routine, * we use a second flag to acknowledge that we came * from an idle state with ticks stopped. */ @@ -482,7 +507,7 @@ void rcu_irq_enter(void) /* * If we take an NMI/SMI now, they will also increment * the rcu_update_flag, and will not update the - * dynticks_progress_counter on exit. That is for + * rcu_dyntick_sched.dynticks on exit. That is for * this IRQ to do. */ } @@ -492,12 +517,13 @@ void rcu_irq_enter(void) * rcu_irq_exit - Called from exiting Hard irq context. * * If the CPU was idle with dynamic ticks active, update the - * dynticks_progress_counter to put let the RCU handling be + * rcu_dyntick_sched.dynticks to put let the RCU handling be * aware that the CPU is going back to idle with no ticks. */ void rcu_irq_exit(void) { int cpu = smp_processor_id(); + struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu); /* * rcu_update_flag is set if we interrupted the CPU @@ -505,7 +531,7 @@ void rcu_irq_exit(void) * Once this occurs, we keep track of interrupt nesting * because a NMI/SMI could also come in, and we still * only want the IRQ that started the increment of the - * dynticks_progress_counter to be the one that modifies + * rcu_dyntick_sched.dynticks to be the one that modifies * it on exit. */ if (per_cpu(rcu_update_flag, cpu)) { @@ -517,28 +543,29 @@ void rcu_irq_exit(void) /* * If an NMI/SMI happens now we are still - * protected by the dynticks_progress_counter being odd. + * protected by the rcu_dyntick_sched.dynticks being odd. */ /* * The following memory barrier ensures that any * rcu_read_unlock() primitives in the irq handler * are seen by other CPUs to preceed the following - * increment to dynticks_progress_counter. This + * increment to rcu_dyntick_sched.dynticks. This * is required in order for other CPUs to determine * when it is safe to advance the RCU grace-period * state machine. */ smp_mb(); /* see above block comment. */ - per_cpu(dynticks_progress_counter, cpu)++; - WARN_ON(per_cpu(dynticks_progress_counter, cpu) & 0x1); + rdssp->dynticks++; + WARN_ON(rdssp->dynticks & 0x1); } } static void dyntick_save_progress_counter(int cpu) { - per_cpu(rcu_dyntick_snapshot, cpu) = - per_cpu(dynticks_progress_counter, cpu); + struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu); + + rdssp->dynticks_snap = rdssp->dynticks; } static inline int @@ -546,9 +573,10 @@ rcu_try_flip_waitack_needed(int cpu) { long curr; long snap; + struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu); - curr = per_cpu(dynticks_progress_counter, cpu); - snap = per_cpu(rcu_dyntick_snapshot, cpu); + curr = rdssp->dynticks; + snap = rdssp->dynticks_snap; smp_mb(); /* force ordering with cpu entering/leaving dynticks. */ /* @@ -569,7 +597,7 @@ rcu_try_flip_waitack_needed(int cpu) * that this CPU already acknowledged the counter. */ - if ((curr - snap) > 2 || (snap & 0x1) == 0) + if ((curr - snap) > 2 || (curr & 0x1) == 0) return 0; /* We need this CPU to explicitly acknowledge the counter flip. */ @@ -582,9 +610,10 @@ rcu_try_flip_waitmb_needed(int cpu) { long curr; long snap; + struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu); - curr = per_cpu(dynticks_progress_counter, cpu); - snap = per_cpu(rcu_dyntick_snapshot, cpu); + curr = rdssp->dynticks; + snap = rdssp->dynticks_snap; smp_mb(); /* force ordering with cpu entering/leaving dynticks. */ /* @@ -611,14 +640,86 @@ rcu_try_flip_waitmb_needed(int cpu) return 1; } +static void dyntick_save_progress_counter_sched(int cpu) +{ + struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu); + + rdssp->sched_dynticks_snap = rdssp->dynticks; +} + +static int rcu_qsctr_inc_needed_dyntick(int cpu) +{ + long curr; + long snap; + struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu); + + curr = rdssp->dynticks; + snap = rdssp->sched_dynticks_snap; + smp_mb(); /* force ordering with cpu entering/leaving dynticks. */ + + /* + * If the CPU remained in dynticks mode for the entire time + * and didn't take any interrupts, NMIs, SMIs, or whatever, + * then it cannot be in the middle of an rcu_read_lock(), so + * the next rcu_read_lock() it executes must use the new value + * of the counter. Therefore, this CPU has been in a quiescent + * state the entire time, and we don't need to wait for it. + */ + + if ((curr == snap) && ((curr & 0x1) == 0)) + return 0; + + /* + * If the CPU passed through or entered a dynticks idle phase with + * no active irq handlers, then, as above, this CPU has already + * passed through a quiescent state. + */ + + if ((curr - snap) > 2 || (snap & 0x1) == 0) + return 0; + + /* We need this CPU to go through a quiescent state. */ + + return 1; +} + #else /* !CONFIG_NO_HZ */ -# define dyntick_save_progress_counter(cpu) do { } while (0) -# define rcu_try_flip_waitack_needed(cpu) (1) -# define rcu_try_flip_waitmb_needed(cpu) (1) +# define dyntick_save_progress_counter(cpu) do { } while (0) +# define rcu_try_flip_waitack_needed(cpu) (1) +# define rcu_try_flip_waitmb_needed(cpu) (1) + +# define dyntick_save_progress_counter_sched(cpu) do { } while (0) +# define rcu_qsctr_inc_needed_dyntick(cpu) (1) #endif /* CONFIG_NO_HZ */ +static void save_qsctr_sched(int cpu) +{ + struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu); + + rdssp->sched_qs_snap = rdssp->sched_qs; +} + +static inline int rcu_qsctr_inc_needed(int cpu) +{ + struct rcu_dyntick_sched *rdssp = &per_cpu(rcu_dyntick_sched, cpu); + + /* + * If there has been a quiescent state, no more need to wait + * on this CPU. + */ + + if (rdssp->sched_qs != rdssp->sched_qs_snap) { + smp_mb(); /* force ordering with cpu entering schedule(). */ + return 0; + } + + /* We need this CPU to go through a quiescent state. */ + + return 1; +} + /* * Get here when RCU is idle. Decide whether we need to * move out of idle state, and return non-zero if so. @@ -657,7 +758,7 @@ rcu_try_flip_idle(void) /* Now ask each CPU for acknowledgement of the flip. */ - for_each_cpu_mask(cpu, rcu_cpu_online_map) { + for_each_cpu_mask_nr(cpu, rcu_cpu_online_map) { per_cpu(rcu_flip_flag, cpu) = rcu_flipped; dyntick_save_progress_counter(cpu); } @@ -675,7 +776,7 @@ rcu_try_flip_waitack(void) int cpu; RCU_TRACE_ME(rcupreempt_trace_try_flip_a1); - for_each_cpu_mask(cpu, rcu_cpu_online_map) + for_each_cpu_mask_nr(cpu, rcu_cpu_online_map) if (rcu_try_flip_waitack_needed(cpu) && per_cpu(rcu_flip_flag, cpu) != rcu_flip_seen) { RCU_TRACE_ME(rcupreempt_trace_try_flip_ae1); @@ -707,7 +808,7 @@ rcu_try_flip_waitzero(void) /* Check to see if the sum of the "last" counters is zero. */ RCU_TRACE_ME(rcupreempt_trace_try_flip_z1); - for_each_cpu_mask(cpu, rcu_cpu_online_map) + for_each_cpu_mask_nr(cpu, rcu_cpu_online_map) sum += RCU_DATA_CPU(cpu)->rcu_flipctr[lastidx]; if (sum != 0) { RCU_TRACE_ME(rcupreempt_trace_try_flip_ze1); @@ -722,7 +823,7 @@ rcu_try_flip_waitzero(void) smp_mb(); /* ^^^^^^^^^^^^ */ /* Call for a memory barrier from each CPU. */ - for_each_cpu_mask(cpu, rcu_cpu_online_map) { + for_each_cpu_mask_nr(cpu, rcu_cpu_online_map) { per_cpu(rcu_mb_flag, cpu) = rcu_mb_needed; dyntick_save_progress_counter(cpu); } @@ -742,7 +843,7 @@ rcu_try_flip_waitmb(void) int cpu; RCU_TRACE_ME(rcupreempt_trace_try_flip_m1); - for_each_cpu_mask(cpu, rcu_cpu_online_map) + for_each_cpu_mask_nr(cpu, rcu_cpu_online_map) if (rcu_try_flip_waitmb_needed(cpu) && per_cpu(rcu_mb_flag, cpu) != rcu_mb_done) { RCU_TRACE_ME(rcupreempt_trace_try_flip_me1); @@ -821,6 +922,26 @@ void rcu_check_callbacks(int cpu, int user) unsigned long flags; struct rcu_data *rdp = RCU_DATA_CPU(cpu); + /* + * If this CPU took its interrupt from user mode or from the + * idle loop, and this is not a nested interrupt, then + * this CPU has to have exited all prior preept-disable + * sections of code. So increment the counter to note this. + * + * The memory barrier is needed to handle the case where + * writes from a preempt-disable section of code get reordered + * into schedule() by this CPU's write buffer. So the memory + * barrier makes sure that the rcu_qsctr_inc() is seen by other + * CPUs to happen after any such write. + */ + + if (user || + (idle_cpu(cpu) && !in_softirq() && + hardirq_count() <= (1 << HARDIRQ_SHIFT))) { + smp_mb(); /* Guard against aggressive schedule(). */ + rcu_qsctr_inc(cpu); + } + rcu_check_mb(cpu); if (rcu_ctrlblk.completed == rdp->completed) rcu_try_flip(); @@ -871,6 +992,8 @@ void rcu_offline_cpu(int cpu) struct rcu_head *list = NULL; unsigned long flags; struct rcu_data *rdp = RCU_DATA_CPU(cpu); + struct rcu_head *schedlist = NULL; + struct rcu_head **schedtail = &schedlist; struct rcu_head **tail = &list; /* @@ -884,6 +1007,11 @@ void rcu_offline_cpu(int cpu) rcu_offline_cpu_enqueue(rdp->waitlist[i], rdp->waittail[i], list, tail); rcu_offline_cpu_enqueue(rdp->nextlist, rdp->nexttail, list, tail); + rcu_offline_cpu_enqueue(rdp->waitschedlist, rdp->waitschedtail, + schedlist, schedtail); + rcu_offline_cpu_enqueue(rdp->nextschedlist, rdp->nextschedtail, + schedlist, schedtail); + rdp->rcu_sched_sleeping = 0; spin_unlock_irqrestore(&rdp->lock, flags); rdp->waitlistcount = 0; @@ -918,22 +1046,40 @@ void rcu_offline_cpu(int cpu) * fix. */ - local_irq_save(flags); + local_irq_save(flags); /* disable preempt till we know what lock. */ rdp = RCU_DATA_ME(); spin_lock(&rdp->lock); *rdp->nexttail = list; if (list) rdp->nexttail = tail; + *rdp->nextschedtail = schedlist; + if (schedlist) + rdp->nextschedtail = schedtail; spin_unlock_irqrestore(&rdp->lock, flags); } void __devinit rcu_online_cpu(int cpu) { unsigned long flags; + struct rcu_data *rdp; spin_lock_irqsave(&rcu_ctrlblk.fliplock, flags); cpu_set(cpu, rcu_cpu_online_map); spin_unlock_irqrestore(&rcu_ctrlblk.fliplock, flags); + + /* + * The rcu_sched grace-period processing might have bypassed + * this CPU, given that it was not in the rcu_cpu_online_map + * when the grace-period scan started. This means that the + * grace-period task might sleep. So make sure that if this + * should happen, the first callback posted to this CPU will + * wake up the grace-period task if need be. + */ + + rdp = RCU_DATA_CPU(cpu); + spin_lock_irqsave(&rdp->lock, flags); + rdp->rcu_sched_sleeping = 1; + spin_unlock_irqrestore(&rdp->lock, flags); } #else /* #ifdef CONFIG_HOTPLUG_CPU */ @@ -988,31 +1134,196 @@ void call_rcu(struct rcu_head *head, void (*func)(struct rcu_head *rcu)) *rdp->nexttail = head; rdp->nexttail = &head->next; RCU_TRACE_RDP(rcupreempt_trace_next_add, rdp); - spin_unlock(&rdp->lock); - local_irq_restore(flags); + spin_unlock_irqrestore(&rdp->lock, flags); } EXPORT_SYMBOL_GPL(call_rcu); +void call_rcu_sched(struct rcu_head *head, void (*func)(struct rcu_head *rcu)) +{ + unsigned long flags; + struct rcu_data *rdp; + int wake_gp = 0; + + head->func = func; + head->next = NULL; + local_irq_save(flags); + rdp = RCU_DATA_ME(); + spin_lock(&rdp->lock); + *rdp->nextschedtail = head; + rdp->nextschedtail = &head->next; + if (rdp->rcu_sched_sleeping) { + + /* Grace-period processing might be sleeping... */ + + rdp->rcu_sched_sleeping = 0; + wake_gp = 1; + } + spin_unlock_irqrestore(&rdp->lock, flags); + if (wake_gp) { + + /* Wake up grace-period processing, unless someone beat us. */ + + spin_lock_irqsave(&rcu_ctrlblk.schedlock, flags); + if (rcu_ctrlblk.sched_sleep != rcu_sched_sleeping) + wake_gp = 0; + rcu_ctrlblk.sched_sleep = rcu_sched_not_sleeping; + spin_unlock_irqrestore(&rcu_ctrlblk.schedlock, flags); + if (wake_gp) + wake_up_interruptible(&rcu_ctrlblk.sched_wq); + } +} +EXPORT_SYMBOL_GPL(call_rcu_sched); + /* * Wait until all currently running preempt_disable() code segments * (including hardware-irq-disable segments) complete. Note that * in -rt this does -not- necessarily result in all currently executing * interrupt -handlers- having completed. */ -void __synchronize_sched(void) +synchronize_rcu_xxx(__synchronize_sched, call_rcu_sched) +EXPORT_SYMBOL_GPL(__synchronize_sched); + +/* + * kthread function that manages call_rcu_sched grace periods. + */ +static int rcu_sched_grace_period(void *arg) { - cpumask_t oldmask; + int couldsleep; /* might sleep after current pass. */ + int couldsleepnext = 0; /* might sleep after next pass. */ int cpu; + unsigned long flags; + struct rcu_data *rdp; + int ret; - if (sched_getaffinity(0, &oldmask) < 0) - oldmask = cpu_possible_map; - for_each_online_cpu(cpu) { - sched_setaffinity(0, &cpumask_of_cpu(cpu)); - schedule(); - } - sched_setaffinity(0, &oldmask); + /* + * Each pass through the following loop handles one + * rcu_sched grace period cycle. + */ + do { + /* Save each CPU's current state. */ + + for_each_online_cpu(cpu) { + dyntick_save_progress_counter_sched(cpu); + save_qsctr_sched(cpu); + } + + /* + * Sleep for about an RCU grace-period's worth to + * allow better batching and to consume less CPU. + */ + schedule_timeout_interruptible(RCU_SCHED_BATCH_TIME); + + /* + * If there was nothing to do last time, prepare to + * sleep at the end of the current grace period cycle. + */ + couldsleep = couldsleepnext; + couldsleepnext = 1; + if (couldsleep) { + spin_lock_irqsave(&rcu_ctrlblk.schedlock, flags); + rcu_ctrlblk.sched_sleep = rcu_sched_sleep_prep; + spin_unlock_irqrestore(&rcu_ctrlblk.schedlock, flags); + } + + /* + * Wait on each CPU in turn to have either visited + * a quiescent state or been in dynticks-idle mode. + */ + for_each_online_cpu(cpu) { + while (rcu_qsctr_inc_needed(cpu) && + rcu_qsctr_inc_needed_dyntick(cpu)) { + /* resched_cpu(cpu); @@@ */ + schedule_timeout_interruptible(1); + } + } + + /* Advance callbacks for each CPU. */ + + for_each_online_cpu(cpu) { + + rdp = RCU_DATA_CPU(cpu); + spin_lock_irqsave(&rdp->lock, flags); + + /* + * We are running on this CPU irq-disabled, so no + * CPU can go offline until we re-enable irqs. + * The current CPU might have already gone + * offline (between the for_each_offline_cpu and + * the spin_lock_irqsave), but in that case all its + * callback lists will be empty, so no harm done. + * + * Advance the callbacks! We share normal RCU's + * donelist, since callbacks are invoked the + * same way in either case. + */ + if (rdp->waitschedlist != NULL) { + *rdp->donetail = rdp->waitschedlist; + rdp->donetail = rdp->waitschedtail; + + /* + * Next rcu_check_callbacks() will + * do the required raise_softirq(). + */ + } + if (rdp->nextschedlist != NULL) { + rdp->waitschedlist = rdp->nextschedlist; + rdp->waitschedtail = rdp->nextschedtail; + couldsleep = 0; + couldsleepnext = 0; + } else { + rdp->waitschedlist = NULL; + rdp->waitschedtail = &rdp->waitschedlist; + } + rdp->nextschedlist = NULL; + rdp->nextschedtail = &rdp->nextschedlist; + + /* Mark sleep intention. */ + + rdp->rcu_sched_sleeping = couldsleep; + + spin_unlock_irqrestore(&rdp->lock, flags); + } + + /* If we saw callbacks on the last scan, go deal with them. */ + + if (!couldsleep) + continue; + + /* Attempt to block... */ + + spin_lock_irqsave(&rcu_ctrlblk.schedlock, flags); + if (rcu_ctrlblk.sched_sleep != rcu_sched_sleep_prep) { + + /* + * Someone posted a callback after we scanned. + * Go take care of it. + */ + spin_unlock_irqrestore(&rcu_ctrlblk.schedlock, flags); + couldsleepnext = 0; + continue; + } + + /* Block until the next person posts a callback. */ + + rcu_ctrlblk.sched_sleep = rcu_sched_sleeping; + spin_unlock_irqrestore(&rcu_ctrlblk.schedlock, flags); + ret = 0; + __wait_event_interruptible(rcu_ctrlblk.sched_wq, + rcu_ctrlblk.sched_sleep != rcu_sched_sleeping, + ret); + + /* + * Signals would prevent us from sleeping, and we cannot + * do much with them in any case. So flush them. + */ + if (ret) + flush_signals(current); + couldsleepnext = 0; + + } while (!kthread_should_stop()); + + return (0); } -EXPORT_SYMBOL_GPL(__synchronize_sched); /* * Check to see if any future RCU-related work will need to be done @@ -1029,7 +1340,9 @@ int rcu_needs_cpu(int cpu) return (rdp->donelist != NULL || !!rdp->waitlistcount || - rdp->nextlist != NULL); + rdp->nextlist != NULL || + rdp->nextschedlist != NULL || + rdp->waitschedlist != NULL); } int rcu_pending(int cpu) @@ -1040,7 +1353,9 @@ int rcu_pending(int cpu) if (rdp->donelist != NULL || !!rdp->waitlistcount || - rdp->nextlist != NULL) + rdp->nextlist != NULL || + rdp->nextschedlist != NULL || + rdp->waitschedlist != NULL) return 1; /* The RCU core needs an acknowledgement from this CPU. */ @@ -1107,6 +1422,11 @@ void __init __rcu_init(void) rdp->donetail = &rdp->donelist; rdp->rcu_flipctr[0] = 0; rdp->rcu_flipctr[1] = 0; + rdp->nextschedlist = NULL; + rdp->nextschedtail = &rdp->nextschedlist; + rdp->waitschedlist = NULL; + rdp->waitschedtail = &rdp->waitschedlist; + rdp->rcu_sched_sleeping = 0; } register_cpu_notifier(&rcu_nb); @@ -1129,11 +1449,15 @@ void __init __rcu_init(void) } /* - * Deprecated, use synchronize_rcu() or synchronize_sched() instead. + * Late-boot-time RCU initialization that must wait until after scheduler + * has been initialized. */ -void synchronize_kernel(void) +void __init rcu_init_sched(void) { - synchronize_rcu(); + rcu_sched_grace_period_task = kthread_run(rcu_sched_grace_period, + NULL, + "rcu_sched_grace_period"); + WARN_ON(IS_ERR(rcu_sched_grace_period_task)); } #ifdef CONFIG_RCU_TRACE diff --git a/kernel/sched.c b/kernel/sched.c index eb4d91e0418a..b7ec86d48a79 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -367,17 +367,33 @@ static inline void set_task_rq(struct task_struct *p, unsigned int cpu) { } #endif /* CONFIG_GROUP_SCHED */ +struct cfs_root_rq { + u64 min_vruntime; + + struct rb_root tasks_timeline; + struct rb_node *left_timeline; + + struct sched_entity *next; + +#ifdef CONFIG_FAIR_GROUP_SCHED + struct rb_root tasks_deadline; + struct rb_node *left_deadline; + + s64 avg_vruntime; +#endif + unsigned long nr_queued; + +#ifdef CONFIG_SCHEDSTATS + unsigned long nr_spread_over; + u64 exec_clock; +#endif +}; + /* CFS-related fields in a runqueue */ struct cfs_rq { struct load_weight load; unsigned long nr_running; - u64 exec_clock; - u64 min_vruntime; - - struct rb_root tasks_timeline; - struct rb_node *rb_leftmost; - struct list_head tasks; struct list_head *balance_iterator; @@ -385,9 +401,7 @@ struct cfs_rq { * 'curr' points to currently running entity on this cfs_rq. * It is set to NULL otherwise (i.e when none are currently running). */ - struct sched_entity *curr, *next; - - unsigned long nr_spread_over; + struct sched_entity *curr; #ifdef CONFIG_FAIR_GROUP_SCHED struct rq *rq; /* cpu runqueue to which this cfs_rq is attached */ @@ -532,6 +546,7 @@ struct rq { unsigned long nr_load_updates; u64 nr_switches; + struct cfs_root_rq cfs_root; struct cfs_rq cfs; struct rt_rq rt; @@ -640,6 +655,8 @@ static inline void update_rq_clock(struct rq *rq) rq->clock = sched_clock_cpu(cpu_of(rq)); } +#include "sched_trace.h" + /* * Tunables that become constants when CONFIG_SCHED_DEBUG is off: */ @@ -849,7 +866,8 @@ static DEFINE_PER_CPU(unsigned long long, prev_cpu_time); static DEFINE_SPINLOCK(time_sync_lock); static unsigned long long prev_global_time; -static unsigned long long __sync_cpu_clock(unsigned long long time, int cpu) +static unsigned long long notrace +__sync_cpu_clock(unsigned long long time, int cpu) { /* * We want this inlined, to not get tracer function calls @@ -871,7 +889,7 @@ static unsigned long long __sync_cpu_clock(unsigned long long time, int cpu) return time; } -static unsigned long long __cpu_clock(int cpu) +static unsigned long long notrace __cpu_clock(int cpu) { unsigned long long now; @@ -891,7 +909,7 @@ static unsigned long long __cpu_clock(int cpu) * For kernel-internal use: high-speed (but slightly incorrect) per-cpu * clock constructed from sched_clock(): */ -unsigned long long cpu_clock(int cpu) +unsigned long long notrace cpu_clock(int cpu) { unsigned long long prev_cpu_time, time, delta_time; unsigned long flags; @@ -1612,7 +1630,7 @@ void aggregate_group_weight(struct task_group *tg, struct sched_domain *sd) unsigned long task_weight = 0; int i; - for_each_cpu_mask(i, sd->span) { + for_each_cpu_mask_nr(i, sd->span) { rq_weight += tg->cfs_rq[i]->load.weight; task_weight += tg->cfs_rq[i]->task_weight; } @@ -1630,7 +1648,7 @@ void aggregate_group_shares(struct task_group *tg, struct sched_domain *sd) unsigned long shares = 0; int i; - for_each_cpu_mask(i, sd->span) + for_each_cpu_mask_nr(i, sd->span) shares += tg->cfs_rq[i]->shares; if ((!shares && aggregate(tg, sd)->rq_weight) || shares > tg->shares) @@ -1652,7 +1670,7 @@ void aggregate_group_load(struct task_group *tg, struct sched_domain *sd) int i; load = 0; - for_each_cpu_mask(i, sd->span) + for_each_cpu_mask_nr(i, sd->span) load += cpu_rq(i)->load.weight; } else { @@ -1763,7 +1781,7 @@ void aggregate_group_set_shares(struct task_group *tg, struct sched_domain *sd) unsigned long shares = aggregate(tg, sd)->shares; int i; - for_each_cpu_mask(i, sd->span) { + for_each_cpu_mask_nr(i, sd->span) { struct rq *rq = cpu_rq(i); unsigned long flags; @@ -1995,7 +2013,7 @@ inline int task_curr(const struct task_struct *p) } /* Used instead of source_load when we know the type == 0 */ -unsigned long weighted_cpuload(const int cpu) +static unsigned long weighted_cpuload(const int cpu) { return cpu_rq(cpu)->load.weight; } @@ -2039,7 +2057,8 @@ task_hot(struct task_struct *p, u64 now, struct sched_domain *sd) /* * Buddy candidates are cache hot: */ - if (sched_feat(CACHE_HOT_BUDDY) && (&p->se == cfs_rq_of(&p->se)->next)) + if (sched_feat(CACHE_HOT_BUDDY) && + (&p->se == task_rq(p)->cfs_root.next)) return 1; if (p->sched_class != &fair_sched_class) @@ -2060,8 +2079,8 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu) { int old_cpu = task_cpu(p); struct rq *old_rq = cpu_rq(old_cpu), *new_rq = cpu_rq(new_cpu); - struct cfs_rq *old_cfsrq = task_cfs_rq(p), - *new_cfsrq = cpu_cfs_rq(old_cfsrq, new_cpu); + struct cfs_root_rq *old_cfs_r_rq = &old_rq->cfs_root, + *new_cfs_r_rq = &new_rq->cfs_root; u64 clock_offset; clock_offset = old_rq->clock - new_rq->clock; @@ -2079,8 +2098,8 @@ void set_task_cpu(struct task_struct *p, unsigned int new_cpu) schedstat_inc(p, se.nr_forced2_migrations); } #endif - p->se.vruntime -= old_cfsrq->min_vruntime - - new_cfsrq->min_vruntime; + p->se.vruntime -= old_cfs_r_rq->min_vruntime - + new_cfs_r_rq->min_vruntime; __set_task_cpu(p, new_cpu); } @@ -2164,6 +2183,7 @@ void wait_task_inactive(struct task_struct *p) * just go back and repeat. */ rq = task_rq_lock(p, &flags); + trace_kernel_sched_wait(p); running = task_running(rq, p); on_rq = p->se.on_rq; task_rq_unlock(rq, &flags); @@ -2297,7 +2317,7 @@ find_idlest_group(struct sched_domain *sd, struct task_struct *p, int this_cpu) /* Tally up the load of all CPUs in the group */ avg_load = 0; - for_each_cpu_mask(i, group->cpumask) { + for_each_cpu_mask_nr(i, group->cpumask) { /* Bias balancing toward cpus of our domain */ if (local_group) load = source_load(i, load_idx); @@ -2339,7 +2359,7 @@ find_idlest_cpu(struct sched_group *group, struct task_struct *p, int this_cpu, /* Traverse only the allowed CPUs */ cpus_and(*tmp, group->cpumask, p->cpus_allowed); - for_each_cpu_mask(i, *tmp) { + for_each_cpu_mask_nr(i, *tmp) { load = weighted_cpuload(i); if (load < min_load || (load == min_load && i == this_cpu)) { @@ -2507,7 +2527,7 @@ out_activate: success = 1; out_running: - ftrace_wake_up_task(rq, p, rq->curr); + trace_kernel_sched_wakeup(rq, p); check_preempt_curr(rq, p); p->state = TASK_RUNNING; @@ -2638,7 +2658,7 @@ void wake_up_new_task(struct task_struct *p, unsigned long clone_flags) p->sched_class->task_new(rq, p); inc_nr_running(rq); } - ftrace_wake_up_task(rq, p, rq->curr); + trace_kernel_sched_wakeup_new(rq, p); check_preempt_curr(rq, p); #ifdef CONFIG_SMP if (p->sched_class->task_wake_up) @@ -2811,7 +2831,8 @@ context_switch(struct rq *rq, struct task_struct *prev, struct mm_struct *mm, *oldmm; prepare_task_switch(rq, prev, next); - ftrace_ctx_switch(rq, prev, next); + + trace_kernel_sched_switch(rq, prev, next); mm = next->mm; oldmm = prev->active_mm; /* @@ -3044,6 +3065,7 @@ static void sched_migrate_task(struct task_struct *p, int dest_cpu) || unlikely(cpu_is_offline(dest_cpu))) goto out; + trace_kernel_sched_migrate_task(p, cpu_of(rq), dest_cpu); /* force the process onto the specified CPU */ if (migrate_task(p, dest_cpu, &req)) { /* Need to wait for migration thread (might exit: take ref). */ @@ -3325,7 +3347,7 @@ find_busiest_group(struct sched_domain *sd, int this_cpu, max_cpu_load = 0; min_cpu_load = ~0UL; - for_each_cpu_mask(i, group->cpumask) { + for_each_cpu_mask_nr(i, group->cpumask) { struct rq *rq; if (!cpu_isset(i, *cpus)) @@ -3589,7 +3611,7 @@ find_busiest_queue(struct sched_group *group, enum cpu_idle_type idle, unsigned long max_load = 0; int i; - for_each_cpu_mask(i, group->cpumask) { + for_each_cpu_mask_nr(i, group->cpumask) { unsigned long wl; if (!cpu_isset(i, *cpus)) @@ -4042,6 +4064,7 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle) /* Earliest time when we have to do rebalance again */ unsigned long next_balance = jiffies + 60*HZ; int update_next_balance = 0; + int need_serialize; cpumask_t tmp; for_each_domain(cpu, sd) { @@ -4059,8 +4082,9 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle) if (interval > HZ*NR_CPUS/10) interval = HZ*NR_CPUS/10; + need_serialize = sd->flags & SD_SERIALIZE; - if (sd->flags & SD_SERIALIZE) { + if (need_serialize) { if (!spin_trylock(&balancing)) goto out; } @@ -4076,7 +4100,7 @@ static void rebalance_domains(int cpu, enum cpu_idle_type idle) } sd->last_balance = jiffies; } - if (sd->flags & SD_SERIALIZE) + if (need_serialize) spin_unlock(&balancing); out: if (time_after(next_balance, sd->last_balance + interval)) { @@ -4129,7 +4153,7 @@ static void run_rebalance_domains(struct softirq_action *h) int balance_cpu; cpu_clear(this_cpu, cpus); - for_each_cpu_mask(balance_cpu, cpus) { + for_each_cpu_mask_nr(balance_cpu, cpus) { /* * If this cpu gets work to do, stop the load balancing * work being done for other cpus. Next load @@ -5496,6 +5520,9 @@ asmlinkage long sys_sched_setaffinity(pid_t pid, unsigned int len, cpumask_t cpu_present_map __read_mostly; EXPORT_SYMBOL(cpu_present_map); +cpumask_t cpu_system_map __read_mostly = CPU_MASK_ALL; +EXPORT_SYMBOL(cpu_system_map); + #ifndef CONFIG_SMP cpumask_t cpu_online_map __read_mostly = CPU_MASK_ALL; EXPORT_SYMBOL(cpu_online_map); @@ -6862,24 +6889,6 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu) rcu_assign_pointer(rq->sd, sd); } -/* cpus with isolated domains */ -static cpumask_t cpu_isolated_map = CPU_MASK_NONE; - -/* Setup the mask of cpus configured for isolated domains */ -static int __init isolated_cpu_setup(char *str) -{ - int ints[NR_CPUS], i; - - str = get_options(str, ARRAY_SIZE(ints), ints); - cpus_clear(cpu_isolated_map); - for (i = 1; i <= ints[0]; i++) - if (ints[i] < NR_CPUS) - cpu_set(ints[i], cpu_isolated_map); - return 1; -} - -__setup("isolcpus=", isolated_cpu_setup); - /* * init_sched_build_groups takes the cpumask we wish to span, and a pointer * to a function which identifies what group(along with sched group) a CPU @@ -6902,7 +6911,7 @@ init_sched_build_groups(const cpumask_t *span, const cpumask_t *cpu_map, cpus_clear(*covered); - for_each_cpu_mask(i, *span) { + for_each_cpu_mask_nr(i, *span) { struct sched_group *sg; int group = group_fn(i, cpu_map, &sg, tmpmask); int j; @@ -6913,7 +6922,7 @@ init_sched_build_groups(const cpumask_t *span, const cpumask_t *cpu_map, cpus_clear(sg->cpumask); sg->__cpu_power = 0; - for_each_cpu_mask(j, *span) { + for_each_cpu_mask_nr(j, *span) { if (group_fn(j, cpu_map, NULL, tmpmask) != group) continue; @@ -6949,9 +6958,9 @@ static int find_next_best_node(int node, nodemask_t *used_nodes) min_val = INT_MAX; - for (i = 0; i < MAX_NUMNODES; i++) { + for (i = 0; i < nr_node_ids; i++) { /* Start at @node */ - n = (node + i) % MAX_NUMNODES; + n = (node + i) % nr_node_ids; if (!nr_cpus_node(n)) continue; @@ -7113,7 +7122,7 @@ static void init_numa_sched_groups_power(struct sched_group *group_head) if (!sg) return; do { - for_each_cpu_mask(j, sg->cpumask) { + for_each_cpu_mask_nr(j, sg->cpumask) { struct sched_domain *sd; sd = &per_cpu(phys_domains, j); @@ -7138,14 +7147,14 @@ static void free_sched_groups(const cpumask_t *cpu_map, cpumask_t *nodemask) { int cpu, i; - for_each_cpu_mask(cpu, *cpu_map) { + for_each_cpu_mask_nr(cpu, *cpu_map) { struct sched_group **sched_group_nodes = sched_group_nodes_bycpu[cpu]; if (!sched_group_nodes) continue; - for (i = 0; i < MAX_NUMNODES; i++) { + for (i = 0; i < nr_node_ids; i++) { struct sched_group *oldsg, *sg = sched_group_nodes[i]; *nodemask = node_to_cpumask(i); @@ -7333,7 +7342,7 @@ static int __build_sched_domains(const cpumask_t *cpu_map, /* * Allocate the per-node list of sched groups */ - sched_group_nodes = kcalloc(MAX_NUMNODES, sizeof(struct sched_group *), + sched_group_nodes = kcalloc(nr_node_ids, sizeof(struct sched_group *), GFP_KERNEL); if (!sched_group_nodes) { printk(KERN_WARNING "Can not alloc sched group node list\n"); @@ -7372,7 +7381,7 @@ static int __build_sched_domains(const cpumask_t *cpu_map, /* * Set up domains for cpus specified by the cpu_map. */ - for_each_cpu_mask(i, *cpu_map) { + for_each_cpu_mask_nr(i, *cpu_map) { struct sched_domain *sd = NULL, *p; SCHED_CPUMASK_VAR(nodemask, allmasks); @@ -7444,7 +7453,7 @@ static int __build_sched_domains(const cpumask_t *cpu_map, #ifdef CONFIG_SCHED_SMT /* Set up CPU (sibling) groups */ - for_each_cpu_mask(i, *cpu_map) { + for_each_cpu_mask_nr(i, *cpu_map) { SCHED_CPUMASK_VAR(this_sibling_map, allmasks); SCHED_CPUMASK_VAR(send_covered, allmasks); @@ -7461,7 +7470,7 @@ static int __build_sched_domains(const cpumask_t *cpu_map, #ifdef CONFIG_SCHED_MC /* Set up multi-core groups */ - for_each_cpu_mask(i, *cpu_map) { + for_each_cpu_mask_nr(i, *cpu_map) { SCHED_CPUMASK_VAR(this_core_map, allmasks); SCHED_CPUMASK_VAR(send_covered, allmasks); @@ -7477,7 +7486,7 @@ static int __build_sched_domains(const cpumask_t *cpu_map, #endif /* Set up physical groups */ - for (i = 0; i < MAX_NUMNODES; i++) { + for (i = 0; i < nr_node_ids; i++) { SCHED_CPUMASK_VAR(nodemask, allmasks); SCHED_CPUMASK_VAR(send_covered, allmasks); @@ -7501,7 +7510,7 @@ static int __build_sched_domains(const cpumask_t *cpu_map, send_covered, tmpmask); } - for (i = 0; i < MAX_NUMNODES; i++) { + for (i = 0; i < nr_node_ids; i++) { /* Set up node groups */ struct sched_group *sg, *prev; SCHED_CPUMASK_VAR(nodemask, allmasks); @@ -7528,7 +7537,7 @@ static int __build_sched_domains(const cpumask_t *cpu_map, goto error; } sched_group_nodes[i] = sg; - for_each_cpu_mask(j, *nodemask) { + for_each_cpu_mask_nr(j, *nodemask) { struct sched_domain *sd; sd = &per_cpu(node_domains, j); @@ -7540,9 +7549,9 @@ static int __build_sched_domains(const cpumask_t *cpu_map, cpus_or(*covered, *covered, *nodemask); prev = sg; - for (j = 0; j < MAX_NUMNODES; j++) { + for (j = 0; j < nr_node_ids; j++) { SCHED_CPUMASK_VAR(notcovered, allmasks); - int n = (i + j) % MAX_NUMNODES; + int n = (i + j) % nr_node_ids; node_to_cpumask_ptr(pnodemask, n); cpus_complement(*notcovered, *covered); @@ -7574,28 +7583,28 @@ static int __build_sched_domains(const cpumask_t *cpu_map, /* Calculate CPU power for physical packages and nodes */ #ifdef CONFIG_SCHED_SMT - for_each_cpu_mask(i, *cpu_map) { + for_each_cpu_mask_nr(i, *cpu_map) { struct sched_domain *sd = &per_cpu(cpu_domains, i); init_sched_groups_power(i, sd); } #endif #ifdef CONFIG_SCHED_MC - for_each_cpu_mask(i, *cpu_map) { + for_each_cpu_mask_nr(i, *cpu_map) { struct sched_domain *sd = &per_cpu(core_domains, i); init_sched_groups_power(i, sd); } #endif - for_each_cpu_mask(i, *cpu_map) { + for_each_cpu_mask_nr(i, *cpu_map) { struct sched_domain *sd = &per_cpu(phys_domains, i); init_sched_groups_power(i, sd); } #ifdef CONFIG_NUMA - for (i = 0; i < MAX_NUMNODES; i++) + for (i = 0; i < nr_node_ids; i++) init_numa_sched_groups_power(sched_group_nodes[i]); if (sd_allnodes) { @@ -7608,7 +7617,7 @@ static int __build_sched_domains(const cpumask_t *cpu_map, #endif /* Attach the domains */ - for_each_cpu_mask(i, *cpu_map) { + for_each_cpu_mask_nr(i, *cpu_map) { struct sched_domain *sd; #ifdef CONFIG_SCHED_SMT sd = &per_cpu(cpu_domains, i); @@ -7666,7 +7675,7 @@ static int arch_init_sched_domains(const cpumask_t *cpu_map) doms_cur = kmalloc(sizeof(cpumask_t), GFP_KERNEL); if (!doms_cur) doms_cur = &fallback_doms; - cpus_andnot(*doms_cur, *cpu_map, cpu_isolated_map); + *doms_cur = *cpu_map; dattr_cur = NULL; err = build_sched_domains(doms_cur); register_sched_domain_sysctl(); @@ -7691,7 +7700,7 @@ static void detach_destroy_domains(const cpumask_t *cpu_map) unregister_sched_domain_sysctl(); - for_each_cpu_mask(i, *cpu_map) + for_each_cpu_mask_nr(i, *cpu_map) cpu_attach_domain(NULL, &def_root_domain, i); synchronize_sched(); arch_destroy_sched_domains(cpu_map, &tmpmask); @@ -7747,7 +7756,7 @@ void partition_sched_domains(int ndoms_new, cpumask_t *doms_new, if (doms_new == NULL) { ndoms_new = 1; doms_new = &fallback_doms; - cpus_andnot(doms_new[0], cpu_online_map, cpu_isolated_map); + doms_new[0] = cpu_online_map; dattr_new = NULL; } @@ -7920,7 +7929,7 @@ void __init sched_init_smp(void) get_online_cpus(); mutex_lock(&sched_domains_mutex); arch_init_sched_domains(&cpu_online_map); - cpus_andnot(non_isolated_cpus, cpu_possible_map, cpu_isolated_map); + non_isolated_cpus = cpu_possible_map; if (cpus_empty(non_isolated_cpus)) cpu_set(smp_processor_id(), non_isolated_cpus); mutex_unlock(&sched_domains_mutex); @@ -7948,14 +7957,21 @@ int in_sched_functions(unsigned long addr) && addr < (unsigned long)__sched_text_end); } +static void init_cfs_root_rq(struct cfs_root_rq *cfs_r_rq) +{ + cfs_r_rq->tasks_timeline = RB_ROOT; + cfs_r_rq->min_vruntime = (u64)(-(1LL << 20)); +#ifdef CONFIG_FAIR_GROUP_SCHED + cfs_r_rq->tasks_deadline = RB_ROOT; +#endif +} + static void init_cfs_rq(struct cfs_rq *cfs_rq, struct rq *rq) { - cfs_rq->tasks_timeline = RB_ROOT; INIT_LIST_HEAD(&cfs_rq->tasks); #ifdef CONFIG_FAIR_GROUP_SCHED cfs_rq->rq = rq; #endif - cfs_rq->min_vruntime = (u64)(-(1LL << 20)); } static void init_rt_rq(struct rt_rq *rt_rq, struct rq *rq) @@ -8140,6 +8156,7 @@ void __init sched_init(void) spin_lock_init(&rq->lock); lockdep_set_class(&rq->lock, &rq->rq_lock_key); rq->nr_running = 0; + init_cfs_root_rq(&rq->cfs_root); init_cfs_rq(&rq->cfs, rq); init_rt_rq(&rq->rt, rq); #ifdef CONFIG_FAIR_GROUP_SCHED @@ -8656,11 +8673,6 @@ void sched_move_task(struct task_struct *tsk) set_task_rq(tsk, task_cpu(tsk)); -#ifdef CONFIG_FAIR_GROUP_SCHED - if (tsk->sched_class->moved_group) - tsk->sched_class->moved_group(tsk); -#endif - if (unlikely(running)) tsk->sched_class->set_curr_task(rq); if (on_rq) diff --git a/kernel/sched_clock.c b/kernel/sched_clock.c index 9c597e37f7de..4ed20523eb73 100644 --- a/kernel/sched_clock.c +++ b/kernel/sched_clock.c @@ -28,6 +28,7 @@ #include <linux/spinlock.h> #include <linux/ktime.h> #include <linux/module.h> +#include <linux/hardirq.h> #ifdef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK @@ -136,6 +137,13 @@ u64 sched_clock_cpu(int cpu) struct sched_clock_data *scd = cpu_sdc(cpu); u64 now, clock; + /* + * Normally this is not called in NMI context - but if it is, + * trying to do any locking here is totally lethal. + */ + if (unlikely(in_nmi())) + return scd->clock; + WARN_ON_ONCE(!irqs_disabled()); now = sched_clock(); diff --git a/kernel/sched_debug.c b/kernel/sched_debug.c index 5f06118fbc31..356973a4b79c 100644 --- a/kernel/sched_debug.c +++ b/kernel/sched_debug.c @@ -61,9 +61,18 @@ print_task(struct seq_file *m, struct rq *rq, struct task_struct *p) else SEQ_printf(m, " "); - SEQ_printf(m, "%15s %5d %9Ld.%06ld %9Ld %5d ", + SEQ_printf(m, "%15s %5d %9Ld.%06ld%c %9Ld.%06ld%c %9Ld %5d ", p->comm, p->pid, SPLIT_NS(p->se.vruntime), +#ifdef CONFIG_FAIR_GROUP_SCHED + entity_eligible(&rq->cfs_root, &p->se) ? 'e' : ' ', + SPLIT_NS(p->se.deadline), + entity_expired(&rq->cfs_root, &p->se) ? 'x' : ' ', +#else + ' ', + SPLIT_NS(0ULL), + ' ', +#endif (long long)(p->nvcsw + p->nivcsw), p->prio); #ifdef CONFIG_SCHEDSTATS @@ -94,10 +103,12 @@ static void print_rq(struct seq_file *m, struct rq *rq, int rq_cpu) SEQ_printf(m, "\nrunnable tasks:\n" - " task PID tree-key switches prio" - " exec-runtime sum-exec sum-sleep\n" - "------------------------------------------------------" - "----------------------------------------------------\n"); + " task PID timeline-key deadline-key " + " switches prio exec-runtime sum-exec" + " sum-sleep\n" + "----------------------------------------------------------" + "--------------------------------------------------" + "-----------------\n"); read_lock_irqsave(&tasklist_lock, flags); @@ -111,62 +122,78 @@ static void print_rq(struct seq_file *m, struct rq *rq, int rq_cpu) read_unlock_irqrestore(&tasklist_lock, flags); } -void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq) +void print_cfs_root(struct seq_file *m, int cpu) { s64 MIN_vruntime = -1, min_vruntime, max_vruntime = -1, spread, rq0_min_vruntime, spread0; struct rq *rq = &per_cpu(runqueues, cpu); + struct cfs_root_rq *cfs_r_rq = &rq->cfs_root; struct sched_entity *last; unsigned long flags; -#if !defined(CONFIG_CGROUP_SCHED) || !defined(CONFIG_USER_SCHED) - SEQ_printf(m, "\ncfs_rq[%d]:\n", cpu); -#else - char path[128] = ""; - struct cgroup *cgroup = NULL; - struct task_group *tg = cfs_rq->tg; - - if (tg) - cgroup = tg->css.cgroup; - - if (cgroup) - cgroup_path(cgroup, path, sizeof(path)); - - SEQ_printf(m, "\ncfs_rq[%d]:%s\n", cpu, path); -#endif - - SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "exec_clock", - SPLIT_NS(cfs_rq->exec_clock)); + SEQ_printf(m, "\ncfs_root_rq\n"); spin_lock_irqsave(&rq->lock, flags); - if (cfs_rq->rb_leftmost) - MIN_vruntime = (__pick_next_entity(cfs_rq))->vruntime; - last = __pick_last_entity(cfs_rq); + if (cfs_r_rq->left_timeline) + MIN_vruntime = (__pick_next_timeline(cfs_r_rq))->vruntime; + last = __pick_last_timeline(cfs_r_rq); if (last) max_vruntime = last->vruntime; - min_vruntime = rq->cfs.min_vruntime; - rq0_min_vruntime = per_cpu(runqueues, 0).cfs.min_vruntime; + min_vruntime = cfs_r_rq->min_vruntime; + rq0_min_vruntime = per_cpu(runqueues, 0).cfs_root.min_vruntime; spin_unlock_irqrestore(&rq->lock, flags); + SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "MIN_vruntime", SPLIT_NS(MIN_vruntime)); SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "min_vruntime", SPLIT_NS(min_vruntime)); SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "max_vruntime", SPLIT_NS(max_vruntime)); + spread = max_vruntime - MIN_vruntime; SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "spread", SPLIT_NS(spread)); spread0 = min_vruntime - rq0_min_vruntime; SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "spread0", SPLIT_NS(spread0)); + +#ifdef CONFIG_FAIR_GROUP_SCHED + SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "avg_vruntime", + SPLIT_NS(avg_vruntime(cfs_r_rq))); +#endif + +#ifdef CONFIG_SCHEDSTATS + SEQ_printf(m, " .%-30s: %Ld.%06ld\n", "exec_clock", + SPLIT_NS(cfs_r_rq->exec_clock)); + SEQ_printf(m, " .%-30s: %ld\n", "nr_spread_over", + cfs_r_rq->nr_spread_over); +#endif +} + +void print_cfs_rq(struct seq_file *m, int cpu, struct cfs_rq *cfs_rq) +{ +#if !defined(CONFIG_CGROUP_SCHED) || !defined(CONFIG_USER_SCHED) + SEQ_printf(m, "\ncfs_rq[%d]:\n", cpu); +#else + char path[128] = ""; + struct cgroup *cgroup = NULL; + struct task_group *tg = cfs_rq->tg; + + if (tg) + cgroup = tg->css.cgroup; + + if (cgroup) + cgroup_path(cgroup, path, sizeof(path)); + + SEQ_printf(m, "\ncfs_rq[%d]:%s\n", cpu, path); +#endif + SEQ_printf(m, " .%-30s: %ld\n", "nr_running", cfs_rq->nr_running); SEQ_printf(m, " .%-30s: %ld\n", "load", cfs_rq->load.weight); + #ifdef CONFIG_SCHEDSTATS - SEQ_printf(m, " .%-30s: %d\n", "bkl_count", - rq->bkl_count); + SEQ_printf(m, " .%-30s: %d\n", "bkl_count", rq_of(cfs_rq)->bkl_count); #endif - SEQ_printf(m, " .%-30s: %ld\n", "nr_spread_over", - cfs_rq->nr_spread_over); #ifdef CONFIG_FAIR_GROUP_SCHED #ifdef CONFIG_SMP SEQ_printf(m, " .%-30s: %lu\n", "shares", cfs_rq->shares); @@ -212,6 +239,8 @@ static void print_cpu(struct seq_file *m, int cpu) #undef P #undef PN + print_cfs_root(m, cpu); + print_cfs_stats(m, cpu); print_rq(m, rq, cpu); diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c index c863663d204d..e8c289ab7d10 100644 --- a/kernel/sched_fair.c +++ b/kernel/sched_fair.c @@ -216,33 +216,432 @@ static inline u64 min_vruntime(u64 min_vruntime, u64 vruntime) return min_vruntime; } -static inline s64 entity_key(struct cfs_rq *cfs_rq, struct sched_entity *se) +static inline +s64 entity_timeline_key(struct cfs_root_rq *cfs_r_rq, struct sched_entity *se) { - return se->vruntime - cfs_rq->min_vruntime; + return se->vruntime - cfs_r_rq->min_vruntime; } +static inline struct rb_node *first_fair(struct cfs_root_rq *cfs_r_rq) +{ + return cfs_r_rq->left_timeline; +} + +static struct sched_entity *__pick_next_timeline(struct cfs_root_rq *cfs_r_rq) +{ + return rb_entry(first_fair(cfs_r_rq), + struct sched_entity, timeline_node); +} + +static inline +struct sched_entity *__pick_last_timeline(struct cfs_root_rq *cfs_r_rq) +{ + struct rb_node *last = rb_last(&cfs_r_rq->tasks_timeline); + + if (!last) + return NULL; + + return rb_entry(last, struct sched_entity, timeline_node); +} + +#ifdef CONFIG_FAIR_GROUP_SCHED +static inline +void avg_vruntime_add(struct cfs_root_rq *cfs_r_rq, struct sched_entity *se) +{ + s64 key = entity_timeline_key(cfs_r_rq, se); + cfs_r_rq->avg_vruntime += key; + cfs_r_rq->nr_queued++; +} + +static inline +void avg_vruntime_sub(struct cfs_root_rq *cfs_r_rq, struct sched_entity *se) +{ + s64 key = entity_timeline_key(cfs_r_rq, se); + cfs_r_rq->avg_vruntime -= key; + cfs_r_rq->nr_queued--; +} + +static inline +void avg_vruntime_update(struct cfs_root_rq *cfs_r_rq, s64 delta) +{ + cfs_r_rq->avg_vruntime -= cfs_r_rq->nr_queued * delta; +} + +static inline +u64 avg_vruntime(struct cfs_root_rq *cfs_r_rq) +{ + s64 avg = cfs_r_rq->avg_vruntime; + int sign = 0; + + if (avg < 0) { + sign = 1; + avg = -avg; + } + + if (cfs_r_rq->nr_queued) + do_div(avg, cfs_r_rq->nr_queued); + + if (sign) + avg = -avg; + + return cfs_r_rq->min_vruntime + avg; +} + +static inline +int entity_eligible(struct cfs_root_rq *cfs_r_rq, struct sched_entity *se) +{ + s64 vruntime = entity_timeline_key(cfs_r_rq, se); + + return (vruntime * cfs_r_rq->nr_queued) <= cfs_r_rq->avg_vruntime; +} + +static inline +s64 entity_deadline_key(struct cfs_root_rq *cfs_r_rq, struct sched_entity *se) +{ + struct rq *rq = container_of(cfs_r_rq, struct rq, cfs_root); + + return se->deadline - rq->clock; +} + +static inline +int entity_expired(struct cfs_root_rq *cfs_r_rq, struct sched_entity *se) +{ + return entity_deadline_key(cfs_r_rq, se) <= 0; +} + +static u64 sched_vslice_add(struct cfs_rq *cfs_rq, struct sched_entity *se); + +static u64 sched_se_period(struct sched_entity *se) +{ + return sched_vslice_add(cfs_rq_of(se), se); +} + +static void sched_calc_deadline(struct cfs_root_rq *cfs_r_rq, + struct sched_entity *se) +{ + struct rq *rq = container_of(cfs_r_rq, struct rq, cfs_root); + + se->deadline = rq->clock + sched_se_period(se); +} + +static void +__enqueue_deadline(struct cfs_root_rq *cfs_r_rq, struct sched_entity *se) +{ + struct rb_node **link; + struct rb_node *parent = NULL; + struct sched_entity *entry; + s64 key, entry_key; + int leftmost = 1; + + sched_calc_deadline(cfs_r_rq, se); + + link = &cfs_r_rq->tasks_deadline.rb_node; + key = entity_deadline_key(cfs_r_rq, se); + /* + * Find the right place in the rbtree: + */ + while (*link) { + parent = *link; + entry = rb_entry(parent, struct sched_entity, deadline_node); + /* + * Prefer shorter latency tasks over higher. + */ + entry_key = entity_deadline_key(cfs_r_rq, entry); + if (key < entry_key || (key == entry_key && + sched_se_period(se) < sched_se_period(entry))) { + + link = &parent->rb_left; + + } else { + link = &parent->rb_right; + leftmost = 0; + } + } + + /* + * Maintain a cache of leftmost tree entries (it is frequently + * used): + */ + if (leftmost) + cfs_r_rq->left_deadline = &se->deadline_node; + + rb_link_node(&se->deadline_node, parent, link); + rb_insert_color(&se->deadline_node, &cfs_r_rq->tasks_deadline); +} + +static void +__dequeue_deadline(struct cfs_root_rq *cfs_r_rq, struct sched_entity *se) +{ + if (cfs_r_rq->left_deadline == &se->deadline_node) + cfs_r_rq->left_deadline = rb_next(&se->deadline_node); + + rb_erase(&se->deadline_node, &cfs_r_rq->tasks_deadline); +} + +static inline struct rb_node *first_deadline(struct cfs_root_rq *cfs_r_rq) +{ + return cfs_r_rq->left_deadline; +} + +static struct sched_entity *__pick_next_deadline(struct cfs_root_rq *cfs_r_rq) +{ + return rb_entry(first_deadline(cfs_r_rq), + struct sched_entity, deadline_node); +} + +static inline struct sched_entity *se_of(struct rb_node *node) +{ + return rb_entry(node, struct sched_entity, timeline_node); +} + +#define deadline_gt(cfs_r_rq, field, lnode, rnode) \ +({ \ + struct rq *rq = container_of(cfs_r_rq, struct rq, cfs_root); \ + s64 l = se_of(lnode)->field - rq->clock; \ + s64 r = se_of(rnode)->field - rq->clock; \ + l > r; \ +}) + +static struct sched_entity *__pick_next_eevdf(struct cfs_root_rq *cfs_r_rq) +{ + struct rb_node *node = cfs_r_rq->tasks_timeline.rb_node; + struct rb_node *tree = NULL, *path = NULL; + + while (node) { + if (entity_eligible(cfs_r_rq, se_of(node))) { + if (!path || deadline_gt(cfs_r_rq, deadline, path, node)) + path = node; + + if (!tree || (node->rb_left && + deadline_gt(cfs_r_rq, min_deadline, tree, + node->rb_left))) + tree = node->rb_left; + + node = node->rb_right; + } else + node = node->rb_left; + } + + if (!tree || deadline_gt(cfs_r_rq, min_deadline, tree, path)) + return se_of(path); + + for (node = tree; node; ) { + if (se_of(tree)->min_deadline == se_of(node)->min_deadline) + return se_of(node); + + if (node->rb_left && (se_of(node)->min_deadline == + se_of(node->rb_left)->min_deadline)) + node = node->rb_left; + else + node = node->rb_right; + } + + BUG(); + + return NULL; +} + +static struct sched_entity *__pick_next_entity(struct cfs_root_rq *cfs_r_rq) +{ + struct sched_entity *next; + + if (sched_feat(EEVDF)) { + next = __pick_next_eevdf(cfs_r_rq); + next->eligible = 1; + + } else if (sched_feat(DEADLINE)) { + next = __pick_next_deadline(cfs_r_rq); + + next->eligible = entity_eligible(cfs_r_rq, next); + if (next->eligible || entity_expired(cfs_r_rq, next)) + return next; + } + + next = __pick_next_timeline(cfs_r_rq); + next->eligible = 1; + + return next; +} + +static void update_min_deadline(struct cfs_root_rq *cfs_r_rq, + struct sched_entity *se, struct rb_node *node) +{ + if (node) { + struct sched_entity *child = rb_entry(node, + struct sched_entity, timeline_node); + struct rq *rq = container_of(cfs_r_rq, struct rq, cfs_root); + + if ((se->min_deadline - rq->clock) > + (child->min_deadline - rq->clock)) + se->min_deadline = child->min_deadline; + } +} + +static void update_node(struct cfs_root_rq *cfs_r_rq, struct rb_node *node) +{ + struct sched_entity *se = rb_entry(node, + struct sched_entity, timeline_node); + + se->min_deadline = se->deadline; + update_min_deadline(cfs_r_rq, se, node->rb_right); + update_min_deadline(cfs_r_rq, se, node->rb_left); +} + +static void update_tree(struct cfs_root_rq *cfs_r_rq, struct rb_node *node) +{ + struct rb_node *parent; +up: + update_node(cfs_r_rq, node); + + parent = rb_parent(node); + if (!parent) + return; + + if (node == parent->rb_left && parent->rb_right) + update_node(cfs_r_rq, parent->rb_right); + else if (parent->rb_left) + update_node(cfs_r_rq, parent->rb_left); + + node = parent; + goto up; +} + +static void +update_tree_enqueue(struct cfs_root_rq *cfs_r_rq, struct sched_entity *se) +{ + struct rb_node *node = &se->timeline_node; + + if (node->rb_left) + node = node->rb_left; + else if (node->rb_right) + node = node->rb_right; + + update_tree(cfs_r_rq, node); +} + +static struct rb_node * +update_tree_dequeue_begin(struct cfs_root_rq *cfs_r_rq, struct sched_entity *se) +{ + struct rb_node *deepest; + struct rb_node *node = &se->timeline_node; + + if (!node->rb_right && !node->rb_left) + deepest = rb_parent(node); + else if (!node->rb_right) + deepest = node->rb_left; + else if (!node->rb_left) + deepest = node->rb_right; + else { + deepest = rb_next(node); + if (deepest->rb_right) + deepest = deepest->rb_right; + else if (rb_parent(deepest) != node) + deepest = rb_parent(deepest); + } + + return deepest; +} + +static void +update_tree_dequeue_end(struct cfs_root_rq *cfs_r_rq, struct rb_node *node) +{ + if (node) + update_tree(cfs_r_rq, node); +} + +#else /* CONFIG_FAIR_GROUP_SCHED */ + +struct cfs_root_rq; + +static inline +void avg_vruntime_add(struct cfs_root_rq *cfs_r_rq, struct sched_entity *se) +{ + cfs_r_rq->nr_queued++; +} + +static inline +void avg_vruntime_sub(struct cfs_root_rq *cfs_r_rq, struct sched_entity *se) +{ + cfs_r_rq->nr_queued--; +} + +static inline +void avg_vruntime_update(struct cfs_root_rq *cfs_r_rq, s64 delta) +{ +} + +static inline +void __enqueue_deadline(struct cfs_root_rq *cfs_r_rq, struct sched_entity *se) +{ +} + +static inline +void __dequeue_deadline(struct cfs_root_rq *cfs_r_rq, struct sched_entity *se) +{ +} + +static inline +void update_tree_enqueue(struct cfs_root_rq *cfs_r_rq, struct sched_entity *se) +{ +} + +static struct rb_node * +update_tree_dequeue_begin(struct cfs_root_rq *cfs_r_rq, struct sched_entity *se) +{ + return NULL; +} + +static void +update_tree_dequeue_end(struct cfs_root_rq *cfs_r_rq, struct rb_node *node) +{ +} + +static inline +struct sched_entity *__pick_next_entity(struct cfs_root_rq *cfs_r_rq) +{ + return __pick_next_timeline(cfs_r_rq); +} +#endif /* CONFIG_FAIR_GROUP_SCHED */ + /* - * Enqueue an entity into the rb-tree: + * maintain cfs_rq->min_vruntime to be a monotonic increasing + * value tracking the leftmost vruntime in the tree. */ -static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) +static void +update_min_vruntime(struct cfs_root_rq *cfs_r_rq, struct sched_entity *se) { - struct rb_node **link = &cfs_rq->tasks_timeline.rb_node; + /* + * open coded max_vruntime() to allow updating avg_vruntime + */ + s64 delta = (s64)(se->vruntime - cfs_r_rq->min_vruntime); + if (delta > 0) { + avg_vruntime_update(cfs_r_rq, delta); + cfs_r_rq->min_vruntime = se->vruntime; + } +} + +static void +__enqueue_timeline(struct cfs_root_rq *cfs_r_rq, struct sched_entity *se) +{ + struct rb_node **link; struct rb_node *parent = NULL; struct sched_entity *entry; - s64 key = entity_key(cfs_rq, se); + s64 key; int leftmost = 1; + link = &cfs_r_rq->tasks_timeline.rb_node; + key = entity_timeline_key(cfs_r_rq, se); /* * Find the right place in the rbtree: */ while (*link) { parent = *link; - entry = rb_entry(parent, struct sched_entity, run_node); + entry = rb_entry(parent, struct sched_entity, timeline_node); /* * We dont care about collisions. Nodes with * the same key stay together. */ - if (key < entity_key(cfs_rq, entry)) { + if (key < entity_timeline_key(cfs_r_rq, entry)) { link = &parent->rb_left; } else { link = &parent->rb_right; @@ -255,61 +654,68 @@ static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) * used): */ if (leftmost) { - cfs_rq->rb_leftmost = &se->run_node; - /* - * maintain cfs_rq->min_vruntime to be a monotonic increasing - * value tracking the leftmost vruntime in the tree. - */ - cfs_rq->min_vruntime = - max_vruntime(cfs_rq->min_vruntime, se->vruntime); + cfs_r_rq->left_timeline = &se->timeline_node; + update_min_vruntime(cfs_r_rq, se); } - rb_link_node(&se->run_node, parent, link); - rb_insert_color(&se->run_node, &cfs_rq->tasks_timeline); + rb_link_node(&se->timeline_node, parent, link); + rb_insert_color(&se->timeline_node, &cfs_r_rq->tasks_timeline); + + update_tree_enqueue(cfs_r_rq, se); } -static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) +static void +__dequeue_timeline(struct cfs_root_rq *cfs_r_rq, struct sched_entity *se) { - if (cfs_rq->rb_leftmost == &se->run_node) { + struct rb_node *node = update_tree_dequeue_begin(cfs_r_rq, se); + + if (cfs_r_rq->left_timeline == &se->timeline_node) { struct rb_node *next_node; - struct sched_entity *next; - next_node = rb_next(&se->run_node); - cfs_rq->rb_leftmost = next_node; + next_node = rb_next(&se->timeline_node); + cfs_r_rq->left_timeline = next_node; if (next_node) { - next = rb_entry(next_node, - struct sched_entity, run_node); - cfs_rq->min_vruntime = - max_vruntime(cfs_rq->min_vruntime, - next->vruntime); + update_min_vruntime(cfs_r_rq, rb_entry(next_node, + struct sched_entity, timeline_node)); } } - if (cfs_rq->next == se) - cfs_rq->next = NULL; + if (cfs_r_rq->next == se) + cfs_r_rq->next = NULL; - rb_erase(&se->run_node, &cfs_rq->tasks_timeline); -} + rb_erase(&se->timeline_node, &cfs_r_rq->tasks_timeline); -static inline struct rb_node *first_fair(struct cfs_rq *cfs_rq) -{ - return cfs_rq->rb_leftmost; + update_tree_dequeue_end(cfs_r_rq, node); } -static struct sched_entity *__pick_next_entity(struct cfs_rq *cfs_rq) +/* + * Enqueue an entity into the rb-tree: + */ +static void __enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) { - return rb_entry(first_fair(cfs_rq), struct sched_entity, run_node); + if (!entity_is_task(se)) + return; + + if (se == cfs_rq->curr) + return; + + avg_vruntime_add(&rq_of(cfs_rq)->cfs_root, se); + __enqueue_timeline(&rq_of(cfs_rq)->cfs_root, se); + __enqueue_deadline(&rq_of(cfs_rq)->cfs_root, se); } -static inline struct sched_entity *__pick_last_entity(struct cfs_rq *cfs_rq) +static void __dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) { - struct rb_node *last = rb_last(&cfs_rq->tasks_timeline); + if (!entity_is_task(se)) + return; - if (!last) - return NULL; + if (se == cfs_rq->curr) + return; - return rb_entry(last, struct sched_entity, run_node); + __dequeue_timeline(&rq_of(cfs_rq)->cfs_root, se); + __dequeue_deadline(&rq_of(cfs_rq)->cfs_root, se); + avg_vruntime_sub(&rq_of(cfs_rq)->cfs_root, se); } /************************************************************** @@ -369,7 +775,7 @@ calc_delta_fair(unsigned long delta, struct sched_entity *se) * * p = (nr <= nl) ? l : l*nr/nl */ -static u64 __sched_period(unsigned long nr_running) +static inline u64 __sched_period(unsigned long nr_running) { u64 period = sysctl_sched_latency; unsigned long nr_latency = sched_nr_latency; @@ -390,7 +796,23 @@ static u64 __sched_period(unsigned long nr_running) */ static u64 sched_slice(struct cfs_rq *cfs_rq, struct sched_entity *se) { - return calc_delta_weight(__sched_period(cfs_rq->nr_running), se); + u64 slice = calc_delta_weight(__sched_period(cfs_rq->nr_running), se); + +#ifdef CONFIG_FAIR_GROUP_SCHED + /* + * Limit the max slice length when there is contention (strictly + * speaking we only need to do this when there are tasks of more + * than a single group). This avoids very longs slices of a lightly + * loaded group delaying tasks from another group. + */ + if (rq_of(cfs_rq)->cfs_root.nr_queued) + slice = min_t(u64, slice, sysctl_sched_min_granularity); + + if (!se->eligible) + slice /= 2; +#endif + + return slice; } /* @@ -448,14 +870,11 @@ static inline void __update_curr(struct cfs_rq *cfs_rq, struct sched_entity *curr, unsigned long delta_exec) { - unsigned long delta_exec_weighted; - schedstat_set(curr->exec_max, max((u64)delta_exec, curr->exec_max)); curr->sum_exec_runtime += delta_exec; - schedstat_add(cfs_rq, exec_clock, delta_exec); - delta_exec_weighted = calc_delta_fair(delta_exec, curr); - curr->vruntime += delta_exec_weighted; + schedstat_add(&rq_of(cfs_rq)->cfs_root, exec_clock, delta_exec); + curr->vruntime += calc_delta_fair(delta_exec, curr); } static void update_curr(struct cfs_rq *cfs_rq) @@ -467,6 +886,11 @@ static void update_curr(struct cfs_rq *cfs_rq) if (unlikely(!curr)) return; + if (!entity_is_task(curr)) + return; + + cfs_rq = &rq_of(cfs_rq)->cfs; + /* * Get the amount of time the current task was running * since the last time we changed load (this cannot @@ -477,11 +901,8 @@ static void update_curr(struct cfs_rq *cfs_rq) __update_curr(cfs_rq, curr, delta_exec); curr->exec_start = now; - if (entity_is_task(curr)) { - struct task_struct *curtask = task_of(curr); - - cpuacct_charge(curtask, delta_exec); - } + if (entity_is_task(curr)) + cpuacct_charge(task_of(curr), delta_exec); } static inline void @@ -616,11 +1037,8 @@ static void enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se) * get a milliseconds-range estimation of the amount of * time that the task spent sleeping: */ - if (unlikely(prof_on == SLEEP_PROFILING)) { - - profile_hits(SLEEP_PROFILING, (void *)get_wchan(tsk), + profile_hits(SLEEP_PROFILING, (void *)get_wchan(task_of(se)), delta >> 20); - } account_scheduler_latency(tsk, delta >> 10, 0); } #endif @@ -629,26 +1047,28 @@ static void enqueue_sleeper(struct cfs_rq *cfs_rq, struct sched_entity *se) static void check_spread(struct cfs_rq *cfs_rq, struct sched_entity *se) { #ifdef CONFIG_SCHED_DEBUG - s64 d = se->vruntime - cfs_rq->min_vruntime; + struct cfs_root_rq *cfs_r_rq = &rq_of(cfs_rq)->cfs_root; + s64 d = se->vruntime - cfs_r_rq->min_vruntime; if (d < 0) d = -d; if (d > 3*sysctl_sched_latency) - schedstat_inc(cfs_rq, nr_spread_over); + schedstat_inc(cfs_r_rq, nr_spread_over); #endif } static void place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial) { + struct cfs_root_rq *cfs_r_rq = &rq_of(cfs_rq)->cfs_root; u64 vruntime; - if (first_fair(cfs_rq)) { - vruntime = min_vruntime(cfs_rq->min_vruntime, - __pick_next_entity(cfs_rq)->vruntime); + if (first_fair(cfs_r_rq)) { + vruntime = min_vruntime(cfs_r_rq->min_vruntime, + __pick_next_timeline(cfs_r_rq)->vruntime); } else - vruntime = cfs_rq->min_vruntime; + vruntime = cfs_r_rq->min_vruntime; /* * The 'current' period is already promised to the current tasks, @@ -691,8 +1111,7 @@ enqueue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int wakeup) update_stats_enqueue(cfs_rq, se); check_spread(cfs_rq, se); - if (se != cfs_rq->curr) - __enqueue_entity(cfs_rq, se); + __enqueue_entity(cfs_rq, se); } static void update_avg(u64 *avg, u64 sample) @@ -733,8 +1152,7 @@ dequeue_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int sleep) #endif } - if (se != cfs_rq->curr) - __dequeue_entity(cfs_rq, se); + __dequeue_entity(cfs_rq, se); account_entity_dequeue(cfs_rq, se); } @@ -763,6 +1181,8 @@ set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) * runqueue. */ update_stats_wait_end(cfs_rq, se); + if (WARN_ON_ONCE(cfs_rq->curr)) + cfs_rq->curr = NULL; __dequeue_entity(cfs_rq, se); } @@ -785,31 +1205,6 @@ set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se) static int wakeup_preempt_entity(struct sched_entity *curr, struct sched_entity *se); -static struct sched_entity * -pick_next(struct cfs_rq *cfs_rq, struct sched_entity *se) -{ - if (!cfs_rq->next) - return se; - - if (wakeup_preempt_entity(cfs_rq->next, se) != 0) - return se; - - return cfs_rq->next; -} - -static struct sched_entity *pick_next_entity(struct cfs_rq *cfs_rq) -{ - struct sched_entity *se = NULL; - - if (first_fair(cfs_rq)) { - se = __pick_next_entity(cfs_rq); - se = pick_next(cfs_rq, se); - set_next_entity(cfs_rq, se); - } - - return se; -} - static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev) { /* @@ -820,12 +1215,12 @@ static void put_prev_entity(struct cfs_rq *cfs_rq, struct sched_entity *prev) update_curr(cfs_rq); check_spread(cfs_rq, prev); + cfs_rq->curr = NULL; if (prev->on_rq) { update_stats_wait_start(cfs_rq, prev); /* Put 'current' back into the tree. */ __enqueue_entity(cfs_rq, prev); } - cfs_rq->curr = NULL; } static void @@ -836,6 +1231,9 @@ entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued) */ update_curr(cfs_rq); + if (!entity_is_task(curr)) + return; + #ifdef CONFIG_SCHED_HRTICK /* * queued ticks are scheduled to match the slice, so don't bother @@ -853,7 +1251,8 @@ entity_tick(struct cfs_rq *cfs_rq, struct sched_entity *curr, int queued) return; #endif - if (cfs_rq->nr_running > 1 || !sched_feat(WAKEUP_PREEMPT)) + if (rq_of(cfs_rq)->load.weight != curr->load.weight || + !sched_feat(WAKEUP_PREEMPT)) check_preempt_tick(cfs_rq, curr); } @@ -955,7 +1354,7 @@ static void yield_task_fair(struct rq *rq) /* * Are we the only task in the tree? */ - if (unlikely(cfs_rq->nr_running == 1)) + if (unlikely(!rq->cfs_root.nr_queued)) return; if (likely(!sysctl_sched_compat_yield) && curr->policy != SCHED_BATCH) { @@ -970,7 +1369,7 @@ static void yield_task_fair(struct rq *rq) /* * Find the rightmost entry in the rbtree: */ - rightmost = __pick_last_entity(cfs_rq); + rightmost = __pick_last_timeline(&rq->cfs_root); /* * Already in the rightmost position? */ @@ -1017,7 +1416,7 @@ static int wake_idle(int cpu, struct task_struct *p) || ((sd->flags & SD_WAKE_IDLE_FAR) && !task_hot(p, task_rq(p)->clock, sd))) { cpus_and(tmp, sd->span, p->cpus_allowed); - for_each_cpu_mask(i, tmp) { + for_each_cpu_mask_nr(i, tmp) { if (idle_cpu(i)) { if (i != task_cpu(p)) { schedstat_inc(p, @@ -1201,17 +1600,6 @@ wakeup_preempt_entity(struct sched_entity *curr, struct sched_entity *se) return 0; } -/* return depth at which a sched entity is present in the hierarchy */ -static inline int depth_se(struct sched_entity *se) -{ - int depth = 0; - - for_each_sched_entity(se) - depth++; - - return depth; -} - /* * Preempt the current task with a newly woken task if needed: */ @@ -1220,7 +1608,6 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p) struct task_struct *curr = rq->curr; struct cfs_rq *cfs_rq = task_cfs_rq(curr); struct sched_entity *se = &curr->se, *pse = &p->se; - int se_depth, pse_depth; if (unlikely(rt_prio(p->prio))) { update_rq_clock(rq); @@ -1233,7 +1620,7 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p) if (unlikely(se == pse)) return; - cfs_rq_of(pse)->next = pse; + rq->cfs_root.next = pse; /* * Batch tasks do not preempt (their preemption is driven by @@ -1245,51 +1632,40 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p) if (!sched_feat(WAKEUP_PREEMPT)) return; - /* - * preemption test can be made between sibling entities who are in the - * same cfs_rq i.e who have a common parent. Walk up the hierarchy of - * both tasks until we find their ancestors who are siblings of common - * parent. - */ - - /* First walk up until both entities are at same depth */ - se_depth = depth_se(se); - pse_depth = depth_se(pse); + if (wakeup_preempt_entity(se, pse) == 1) + resched_task(curr); +} - while (se_depth > pse_depth) { - se_depth--; - se = parent_entity(se); - } +static struct sched_entity * +pick_next_entity(struct cfs_root_rq *cfs_r_rq) +{ + struct sched_entity *se = __pick_next_entity(cfs_r_rq); - while (pse_depth > se_depth) { - pse_depth--; - pse = parent_entity(pse); - } + if (!cfs_r_rq->next) + return se; - while (!is_same_group(se, pse)) { - se = parent_entity(se); - pse = parent_entity(pse); - } + if (wakeup_preempt_entity(cfs_r_rq->next, se)) + return se; - if (wakeup_preempt_entity(se, pse) == 1) - resched_task(curr); + return cfs_r_rq->next; } static struct task_struct *pick_next_task_fair(struct rq *rq) { struct task_struct *p; - struct cfs_rq *cfs_rq = &rq->cfs; - struct sched_entity *se; + struct cfs_root_rq *cfs_r_rq = &rq->cfs_root; + struct sched_entity *se, *next; - if (unlikely(!cfs_rq->nr_running)) + if (!first_fair(cfs_r_rq)) return NULL; - do { - se = pick_next_entity(cfs_rq); - cfs_rq = group_cfs_rq(se); - } while (cfs_rq); + next = se = pick_next_entity(cfs_r_rq); + for_each_sched_entity(se) { + struct cfs_rq *cfs_rq = cfs_rq_of(se); + set_next_entity(cfs_rq, se); + } - p = task_of(se); + p = task_of(next); hrtick_start_fair(rq, p); return p; @@ -1301,12 +1677,9 @@ static struct task_struct *pick_next_task_fair(struct rq *rq) static void put_prev_task_fair(struct rq *rq, struct task_struct *prev) { struct sched_entity *se = &prev->se; - struct cfs_rq *cfs_rq; - for_each_sched_entity(se) { - cfs_rq = cfs_rq_of(se); - put_prev_entity(cfs_rq, se); - } + for_each_sched_entity(se) + put_prev_entity(cfs_rq_of(se), se); } #ifdef CONFIG_SMP @@ -1327,23 +1700,18 @@ __load_balance_iterator(struct cfs_rq *cfs_rq, struct list_head *next) struct task_struct *p = NULL; struct sched_entity *se; - if (next == &cfs_rq->tasks) - return NULL; - - /* Skip over entities that are not tasks */ - do { + while (next != &cfs_rq->tasks) { se = list_entry(next, struct sched_entity, group_node); next = next->next; - } while (next != &cfs_rq->tasks && !entity_is_task(se)); - if (next == &cfs_rq->tasks) - return NULL; + /* Skip over entities that are not tasks */ + if (entity_is_task(se)) { + p = task_of(se); + break; + } + } cfs_rq->balance_iterator = next; - - if (entity_is_task(se)) - p = task_of(se); - return p; } @@ -1568,16 +1936,6 @@ static void set_curr_task_fair(struct rq *rq) set_next_entity(cfs_rq_of(se), se); } -#ifdef CONFIG_FAIR_GROUP_SCHED -static void moved_group_fair(struct task_struct *p) -{ - struct cfs_rq *cfs_rq = task_cfs_rq(p); - - update_curr(cfs_rq); - place_entity(cfs_rq, &p->se, 1); -} -#endif - /* * All the scheduling class methods: */ @@ -1606,10 +1964,6 @@ static const struct sched_class fair_sched_class = { .prio_changed = prio_changed_fair, .switched_to = switched_to_fair, - -#ifdef CONFIG_FAIR_GROUP_SCHED - .moved_group = moved_group_fair, -#endif }; #ifdef CONFIG_SCHED_DEBUG diff --git a/kernel/sched_features.h b/kernel/sched_features.h index 1c7283cb9581..23734cef32d7 100644 --- a/kernel/sched_features.h +++ b/kernel/sched_features.h @@ -1,10 +1,11 @@ -SCHED_FEAT(NEW_FAIR_SLEEPERS, 1) +SCHED_FEAT(NEW_FAIR_SLEEPERS, 0) SCHED_FEAT(WAKEUP_PREEMPT, 1) SCHED_FEAT(START_DEBIT, 1) SCHED_FEAT(AFFINE_WAKEUPS, 1) SCHED_FEAT(CACHE_HOT_BUDDY, 1) SCHED_FEAT(SYNC_WAKEUPS, 1) -SCHED_FEAT(HRTICK, 1) +SCHED_FEAT(HRTICK, 0) SCHED_FEAT(DOUBLE_TICK, 0) -SCHED_FEAT(NORMALIZED_SLEEPER, 1) +SCHED_FEAT(NORMALIZED_SLEEPER, 0) SCHED_FEAT(DEADLINE, 1) +SCHED_FEAT(EEVDF, 0) diff --git a/kernel/sched_rt.c b/kernel/sched_rt.c index ea630c20719b..3c3d01e120dc 100644 --- a/kernel/sched_rt.c +++ b/kernel/sched_rt.c @@ -231,7 +231,7 @@ static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun) return 1; span = sched_rt_period_mask(); - for_each_cpu_mask(i, span) { + for_each_cpu_mask_nr(i, span) { int enqueue = 0; struct rt_rq *rt_rq = sched_rt_period_rt_rq(rt_b, i); struct rq *rq = rq_of_rt_rq(rt_rq); @@ -272,7 +272,7 @@ static int balance_runtime(struct rt_rq *rt_rq) spin_lock(&rt_b->rt_runtime_lock); rt_period = ktime_to_ns(rt_b->rt_period); - for_each_cpu_mask(i, rd->span) { + for_each_cpu_mask_nr(i, rd->span) { struct rt_rq *iter = sched_rt_period_rt_rq(rt_b, i); s64 diff; @@ -1013,7 +1013,7 @@ static int pull_rt_task(struct rq *this_rq) next = pick_next_task_rt(this_rq); - for_each_cpu_mask(cpu, this_rq->rd->rto_mask) { + for_each_cpu_mask_nr(cpu, this_rq->rd->rto_mask) { if (this_cpu == cpu) continue; diff --git a/kernel/sched_trace.h b/kernel/sched_trace.h new file mode 100644 index 000000000000..29b48f34fd02 --- /dev/null +++ b/kernel/sched_trace.h @@ -0,0 +1,41 @@ +#include <linux/marker.h> + +static inline void trace_kernel_sched_wait(struct task_struct *p) +{ + trace_mark(kernel_sched_wait_task, "pid %d state %ld", + p->pid, p->state); +} + +static inline +void trace_kernel_sched_wakeup(struct rq *rq, struct task_struct *p) +{ + trace_mark(kernel_sched_wakeup, + "pid %d state %ld ## rq %p task %p rq->curr %p", + p->pid, p->state, rq, p, rq->curr); +} + +static inline +void trace_kernel_sched_wakeup_new(struct rq *rq, struct task_struct *p) +{ + trace_mark(kernel_sched_wakeup_new, + "pid %d state %ld ## rq %p task %p rq->curr %p", + p->pid, p->state, rq, p, rq->curr); +} + +static inline void trace_kernel_sched_switch(struct rq *rq, + struct task_struct *prev, struct task_struct *next) +{ + trace_mark(kernel_sched_schedule, + "prev_pid %d next_pid %d prev_state %ld " + "## rq %p prev %p next %p", + prev->pid, next->pid, prev->state, + rq, prev, next); +} + +static inline void +trace_kernel_sched_migrate_task(struct task_struct *p, int src, int dst) +{ + trace_mark(kernel_sched_migrate_task, + "pid %d state %ld dest_cpu %d", + p->pid, p->state, dst); +} diff --git a/kernel/signal.c b/kernel/signal.c index 72bb4f51f963..09b38f223dd6 100644 --- a/kernel/signal.c +++ b/kernel/signal.c @@ -26,6 +26,7 @@ #include <linux/freezer.h> #include <linux/pid_namespace.h> #include <linux/nsproxy.h> +#include <linux/marker.h> #include <asm/param.h> #include <asm/uaccess.h> @@ -773,6 +774,8 @@ static int send_signal(int sig, struct siginfo *info, struct task_struct *t, struct sigpending *pending; struct sigqueue *q; + trace_mark(kernel_send_signal, "pid %d signal %d", t->pid, sig); + assert_spin_locked(&t->sighand->siglock); if (!prepare_signal(sig, t)) return 0; diff --git a/kernel/softirq.c b/kernel/softirq.c index 36e061740047..0722f93b0df9 100644 --- a/kernel/softirq.c +++ b/kernel/softirq.c @@ -21,6 +21,7 @@ #include <linux/rcupdate.h> #include <linux/smp.h> #include <linux/tick.h> +#include <linux/marker.h> #include <asm/irq.h> /* @@ -231,7 +232,15 @@ restart: do { if (pending & 1) { + trace_mark(kernel_softirq_entry, "softirq_id %lu", + ((unsigned long)h + - (unsigned long)softirq_vec) + / sizeof(*h)); h->action(h); + trace_mark(kernel_softirq_exit, "softirq_id %lu", + ((unsigned long)h + - (unsigned long)softirq_vec) + / sizeof(*h)); rcu_bh_qsctr_inc(cpu); } h++; @@ -323,6 +332,8 @@ void irq_exit(void) */ inline void raise_softirq_irqoff(unsigned int nr) { + trace_mark(kernel_softirq_raise, "softirq_id %u", nr); + __raise_softirq_irqoff(nr); /* @@ -412,7 +423,13 @@ static void tasklet_action(struct softirq_action *a) if (!atomic_read(&t->count)) { if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state)) BUG(); + trace_mark(kernel_tasklet_low_entry, + "func %p data %lu", + t->func, t->data); t->func(t->data); + trace_mark(kernel_tasklet_low_exit, + "func %p data %lu", + t->func, t->data); tasklet_unlock(t); continue; } @@ -447,7 +464,13 @@ static void tasklet_hi_action(struct softirq_action *a) if (!atomic_read(&t->count)) { if (!test_and_clear_bit(TASKLET_STATE_SCHED, &t->state)) BUG(); + trace_mark(kernel_tasklet_high_entry, + "func %p data %lu", + t->func, t->data); t->func(t->data); + trace_mark(kernel_tasklet_high_exit, + "func %p data %lu", + t->func, t->data); tasklet_unlock(t); continue; } diff --git a/kernel/softlockup.c b/kernel/softlockup.c index 01b6522fd92b..1f898abae36d 100644 --- a/kernel/softlockup.c +++ b/kernel/softlockup.c @@ -25,7 +25,22 @@ static DEFINE_PER_CPU(unsigned long, print_timestamp); static DEFINE_PER_CPU(struct task_struct *, watchdog_task); static int __read_mostly did_panic; -unsigned long __read_mostly softlockup_thresh = 60; +int __read_mostly softlockup_thresh = 60; + +/* + * Should we panic (and reboot, if panic_timeout= is set) when a + * soft-lockup occurs: + */ +unsigned int __read_mostly softlockup_panic = + CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC_VALUE; + +static int __init softlockup_panic_setup(char *str) +{ + softlockup_panic = simple_strtoul(str, NULL, 0); + + return 1; +} +__setup("softlockup_panic=", softlockup_panic_setup); static int softlock_panic(struct notifier_block *this, unsigned long event, void *ptr) @@ -79,6 +94,14 @@ void softlockup_tick(void) struct pt_regs *regs = get_irq_regs(); unsigned long now; + /* Is detection switched off? */ + if (!per_cpu(watchdog_task, this_cpu) || softlockup_thresh <= 0) { + /* Be sure we don't false trigger if switched back on */ + if (touch_timestamp) + per_cpu(touch_timestamp, this_cpu) = 0; + return; + } + if (touch_timestamp == 0) { touch_softlockup_watchdog(); return; @@ -89,7 +112,7 @@ void softlockup_tick(void) /* report at most once a second */ if ((print_timestamp >= touch_timestamp && print_timestamp < (touch_timestamp + 1)) || - did_panic || !per_cpu(watchdog_task, this_cpu)) { + did_panic) { return; } @@ -120,6 +143,9 @@ void softlockup_tick(void) else dump_stack(); spin_unlock(&print_lock); + + if (softlockup_panic) + panic("softlockup: hung tasks"); } /* @@ -130,9 +156,9 @@ unsigned long __read_mostly sysctl_hung_task_check_count = 1024; /* * Zero means infinite timeout - no checking done: */ -unsigned long __read_mostly sysctl_hung_task_timeout_secs = 120; +unsigned long __read_mostly sysctl_hung_task_timeout_secs = 240; -unsigned long __read_mostly sysctl_hung_task_warnings = 10; +unsigned long __read_mostly sysctl_hung_task_warnings = 1; /* * Only do the hung-tasks check on one CPU: @@ -172,6 +198,9 @@ static void check_hung_task(struct task_struct *t, unsigned long now) t->last_switch_timestamp = now; touch_nmi_watchdog(); + + if (softlockup_panic) + panic("softlockup: blocked tasks"); } /* @@ -224,7 +253,7 @@ static int watchdog(void *__bind_cpu) */ while (!kthread_should_stop()) { touch_softlockup_watchdog(); - schedule(); + schedule_timeout(softlockup_thresh/2); if (kthread_should_stop()) break; diff --git a/kernel/stacktrace.c b/kernel/stacktrace.c index b71816e47a30..0914d0cbc83c 100644 --- a/kernel/stacktrace.c +++ b/kernel/stacktrace.c @@ -13,6 +13,9 @@ void print_stack_trace(struct stack_trace *trace, int spaces) { int i, j; + if (WARN_ON(!trace->entries)) + return; + for (i = 0; i < trace->nr_entries; i++) { unsigned long ip = trace->entries[i]; diff --git a/kernel/sysctl.c b/kernel/sysctl.c index bf1ebd7b7b87..846bcd32c845 100644 --- a/kernel/sysctl.c +++ b/kernel/sysctl.c @@ -86,12 +86,13 @@ extern int latencytop_enabled; extern unsigned long stopmachine_timeout; /* Constants used for minimum and maximum */ -#if defined(CONFIG_DETECT_SOFTLOCKUP) || defined(CONFIG_HIGHMEM) +#if defined(CONFIG_HIGHMEM) || defined(CONFIG_DETECT_SOFTLOCKUP) static int one = 1; #endif #ifdef CONFIG_DETECT_SOFTLOCKUP static int sixty = 60; +static int neg_one = -1; #endif #ifdef CONFIG_MMU @@ -729,13 +730,24 @@ static struct ctl_table kern_table[] = { #ifdef CONFIG_DETECT_SOFTLOCKUP { .ctl_name = CTL_UNNUMBERED, + .procname = "softlockup_panic", + .data = &softlockup_panic, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = &proc_doulongvec_minmax, + .strategy = &sysctl_intvec, + .extra1 = &zero, + .extra2 = &one, + }, + { + .ctl_name = CTL_UNNUMBERED, .procname = "softlockup_thresh", .data = &softlockup_thresh, - .maxlen = sizeof(unsigned long), + .maxlen = sizeof(int), .mode = 0644, - .proc_handler = &proc_doulongvec_minmax, + .proc_handler = &proc_dointvec_minmax, .strategy = &sysctl_intvec, - .extra1 = &one, + .extra1 = &neg_one, .extra2 = &sixty, }, { diff --git a/kernel/taskstats.c b/kernel/taskstats.c index 4a23517169a6..06b17547f4e7 100644 --- a/kernel/taskstats.c +++ b/kernel/taskstats.c @@ -301,7 +301,7 @@ static int add_del_listener(pid_t pid, cpumask_t *maskp, int isadd) return -EINVAL; if (isadd == REGISTER) { - for_each_cpu_mask(cpu, mask) { + for_each_cpu_mask_nr(cpu, mask) { s = kmalloc_node(sizeof(struct listener), GFP_KERNEL, cpu_to_node(cpu)); if (!s) @@ -320,7 +320,7 @@ static int add_del_listener(pid_t pid, cpumask_t *maskp, int isadd) /* Deregister or cleanup */ cleanup: - for_each_cpu_mask(cpu, mask) { + for_each_cpu_mask_nr(cpu, mask) { listeners = &per_cpu(listener_array, cpu); down_write(&listeners->sem); list_for_each_entry_safe(s, tmp, &listeners->list, list) { diff --git a/kernel/time/clocksource.c b/kernel/time/clocksource.c index dadde5361f32..60ceabd53f2e 100644 --- a/kernel/time/clocksource.c +++ b/kernel/time/clocksource.c @@ -145,9 +145,9 @@ static void clocksource_watchdog(unsigned long data) * Cycle through CPUs to check if the CPUs stay * synchronized to each other. */ - int next_cpu = next_cpu(raw_smp_processor_id(), cpu_online_map); + int next_cpu = next_cpu_nr(raw_smp_processor_id(), cpu_online_map); - if (next_cpu >= NR_CPUS) + if (next_cpu >= nr_cpu_ids) next_cpu = first_cpu(cpu_online_map); watchdog_timer.expires += WATCHDOG_INTERVAL; add_timer_on(&watchdog_timer, next_cpu); diff --git a/kernel/time/tick-broadcast.c b/kernel/time/tick-broadcast.c index 57a1f02e5ec0..2d0a96346259 100644 --- a/kernel/time/tick-broadcast.c +++ b/kernel/time/tick-broadcast.c @@ -397,8 +397,7 @@ again: mask = CPU_MASK_NONE; now = ktime_get(); /* Find all expired events */ - for (cpu = first_cpu(tick_broadcast_oneshot_mask); cpu != NR_CPUS; - cpu = next_cpu(cpu, tick_broadcast_oneshot_mask)) { + for_each_cpu_mask_nr(cpu, tick_broadcast_oneshot_mask) { td = &per_cpu(tick_cpu_device, cpu); if (td->evtdev->next_event.tv64 <= now.tv64) cpu_set(cpu, mask); diff --git a/kernel/timer.c b/kernel/timer.c index ef3fa6906e8f..0185699dc9d9 100644 --- a/kernel/timer.c +++ b/kernel/timer.c @@ -37,12 +37,14 @@ #include <linux/delay.h> #include <linux/tick.h> #include <linux/kallsyms.h> +#include <linux/marker.h> #include <asm/uaccess.h> #include <asm/unistd.h> #include <asm/div64.h> #include <asm/timex.h> #include <asm/io.h> +#include <asm/irq_regs.h> u64 jiffies_64 __cacheline_aligned_in_smp = INITIAL_JIFFIES; @@ -288,6 +290,8 @@ static void internal_add_timer(struct tvec_base *base, struct timer_list *timer) i = (expires >> (TVR_BITS + 3 * TVN_BITS)) & TVN_MASK; vec = base->tv5.vec + i; } + trace_mark(kernel_timer_set, "expires %lu function %p data %lu", + expires, timer->function, timer->data); /* * Timers are FIFO: */ @@ -1066,6 +1070,11 @@ void do_timer(unsigned long ticks) { jiffies_64 += ticks; update_times(ticks); + trace_mark(kernel_timer_update_time, + "jiffies #8u%llu xtime_sec %ld xtime_nsec %ld " + "walltomonotonic_sec %ld walltomonotonic_nsec %ld", + (unsigned long long)jiffies_64, xtime.tv_sec, xtime.tv_nsec, + wall_to_monotonic.tv_sec, wall_to_monotonic.tv_nsec); } #ifdef __ARCH_WANT_SYS_ALARM @@ -1147,7 +1156,9 @@ asmlinkage long sys_getegid(void) static void process_timeout(unsigned long __data) { - wake_up_process((struct task_struct *)__data); + struct task_struct *task = (struct task_struct *)__data; + trace_mark(kernel_timer_timeout, "pid %d", task->pid); + wake_up_process(task); } /** diff --git a/kernel/trace/trace.c b/kernel/trace/trace.c index 01ce5e1794ea..1f91165b3e06 100644 --- a/kernel/trace/trace.c +++ b/kernel/trace/trace.c @@ -27,6 +27,7 @@ #include <linux/poll.h> #include <linux/gfp.h> #include <linux/fs.h> +#include <linux/writeback.h> #include <linux/stacktrace.h> @@ -39,7 +40,7 @@ static unsigned long __read_mostly tracing_nr_buffers; static cpumask_t __read_mostly tracing_buffer_mask; #define for_each_tracing_cpu(cpu) \ - for_each_cpu_mask(cpu, tracing_buffer_mask) + for_each_cpu_mask_nr(cpu, tracing_buffer_mask) /* dummy trace to disable tracing */ static struct tracer no_tracer __read_mostly = { @@ -51,6 +52,8 @@ static int trace_free_page(void); static int tracing_disabled = 1; +static unsigned long tracing_pages_allocated; + long ns2usecs(cycle_t nsec) { @@ -2533,7 +2536,7 @@ tracing_read_pipe(struct file *filp, char __user *ubuf, cpu_set(cpu, mask); } - for_each_cpu_mask(cpu, mask) { + for_each_cpu_mask_nr(cpu, mask) { data = iter->tr->data[cpu]; __raw_spin_lock(&data->lock); @@ -2560,12 +2563,12 @@ tracing_read_pipe(struct file *filp, char __user *ubuf, break; } - for_each_cpu_mask(cpu, mask) { + for_each_cpu_mask_nr(cpu, mask) { data = iter->tr->data[cpu]; __raw_spin_unlock(&data->lock); } - for_each_cpu_mask(cpu, mask) { + for_each_cpu_mask_nr(cpu, mask) { data = iter->tr->data[cpu]; atomic_dec(&data->disabled); } @@ -2633,12 +2636,41 @@ tracing_entries_write(struct file *filp, const char __user *ubuf, } if (val > global_trace.entries) { + long pages_requested; + unsigned long freeable_pages; + + /* make sure we have enough memory before mapping */ + pages_requested = + (val + (ENTRIES_PER_PAGE-1)) / ENTRIES_PER_PAGE; + + /* account for each buffer (and max_tr) */ + pages_requested *= tracing_nr_buffers * 2; + + /* Check for overflow */ + if (pages_requested < 0) { + cnt = -ENOMEM; + goto out; + } + + freeable_pages = determine_dirtyable_memory(); + + /* we only allow to request 1/4 of useable memory */ + if (pages_requested > + ((freeable_pages + tracing_pages_allocated) / 4)) { + cnt = -ENOMEM; + goto out; + } + while (global_trace.entries < val) { if (trace_alloc_page()) { cnt = -ENOMEM; goto out; } + /* double check that we don't go over the known pages */ + if (tracing_pages_allocated > pages_requested) + break; } + } else { /* include the number of entries in val (inc of page entries) */ while (global_trace.entries > val + (ENTRIES_PER_PAGE - 1)) @@ -2821,6 +2853,7 @@ static int trace_alloc_page(void) struct page *page, *tmp; LIST_HEAD(pages); void *array; + unsigned pages_allocated = 0; int i; /* first allocate a page for each CPU */ @@ -2832,6 +2865,7 @@ static int trace_alloc_page(void) goto free_pages; } + pages_allocated++; page = virt_to_page(array); list_add(&page->lru, &pages); @@ -2843,6 +2877,7 @@ static int trace_alloc_page(void) "for trace buffer!\n"); goto free_pages; } + pages_allocated++; page = virt_to_page(array); list_add(&page->lru, &pages); #endif @@ -2864,6 +2899,7 @@ static int trace_alloc_page(void) SetPageLRU(page); #endif } + tracing_pages_allocated += pages_allocated; global_trace.entries += ENTRIES_PER_PAGE; return 0; @@ -2898,6 +2934,8 @@ static int trace_free_page(void) page = list_entry(p, struct page, lru); ClearPageLRU(page); list_del(&page->lru); + tracing_pages_allocated--; + tracing_pages_allocated--; __free_page(page); tracing_reset(data); diff --git a/kernel/trace/trace.h b/kernel/trace/trace.h index 9c0fe319f9b0..3e15c23349e5 100644 --- a/kernel/trace/trace.h +++ b/kernel/trace/trace.h @@ -241,25 +241,10 @@ void update_max_tr_single(struct trace_array *tr, extern cycle_t ftrace_now(int cpu); -#ifdef CONFIG_SCHED_TRACER -extern void -wakeup_sched_switch(struct task_struct *prev, struct task_struct *next); -extern void -wakeup_sched_wakeup(struct task_struct *wakee, struct task_struct *curr); -#else -static inline void -wakeup_sched_switch(struct task_struct *prev, struct task_struct *next) -{ -} -static inline void -wakeup_sched_wakeup(struct task_struct *wakee, struct task_struct *curr) -{ -} -#endif - #ifdef CONFIG_CONTEXT_SWITCH_TRACER typedef void (*tracer_switch_func_t)(void *private, + void *__rq, struct task_struct *prev, struct task_struct *next); @@ -269,9 +254,6 @@ struct tracer_switch_ops { struct tracer_switch_ops *next; }; -extern int register_tracer_switch(struct tracer_switch_ops *ops); -extern int unregister_tracer_switch(struct tracer_switch_ops *ops); - #endif /* CONFIG_CONTEXT_SWITCH_TRACER */ #ifdef CONFIG_DYNAMIC_FTRACE diff --git a/kernel/trace/trace_sched_switch.c b/kernel/trace/trace_sched_switch.c index a3376478fc2c..d25ffa5eaf2b 100644 --- a/kernel/trace/trace_sched_switch.c +++ b/kernel/trace/trace_sched_switch.c @@ -16,11 +16,14 @@ static struct trace_array *ctx_trace; static int __read_mostly tracer_enabled; +static atomic_t sched_ref; static void -ctx_switch_func(void *__rq, struct task_struct *prev, struct task_struct *next) +sched_switch_func(void *private, void *__rq, struct task_struct *prev, + struct task_struct *next) { - struct trace_array *tr = ctx_trace; + struct trace_array **ptr = private; + struct trace_array *tr = *ptr; struct trace_array_cpu *data; unsigned long flags; long disabled; @@ -41,10 +44,40 @@ ctx_switch_func(void *__rq, struct task_struct *prev, struct task_struct *next) local_irq_restore(flags); } +static notrace void +sched_switch_callback(void *probe_data, void *call_data, + const char *format, va_list *args) +{ + struct task_struct *prev; + struct task_struct *next; + struct rq *__rq; + + if (!atomic_read(&sched_ref)) + return; + + /* skip prev_pid %d next_pid %d prev_state %ld */ + (void)va_arg(*args, int); + (void)va_arg(*args, int); + (void)va_arg(*args, long); + __rq = va_arg(*args, typeof(__rq)); + prev = va_arg(*args, typeof(prev)); + next = va_arg(*args, typeof(next)); + + tracing_record_cmdline(prev); + + /* + * If tracer_switch_func only points to the local + * switch func, it still needs the ptr passed to it. + */ + sched_switch_func(probe_data, __rq, prev, next); +} + static void -wakeup_func(void *__rq, struct task_struct *wakee, struct task_struct *curr) +wakeup_func(void *private, void *__rq, struct task_struct *wakee, struct + task_struct *curr) { - struct trace_array *tr = ctx_trace; + struct trace_array **ptr = private; + struct trace_array *tr = *ptr; struct trace_array_cpu *data; unsigned long flags; long disabled; @@ -67,35 +100,29 @@ wakeup_func(void *__rq, struct task_struct *wakee, struct task_struct *curr) local_irq_restore(flags); } -void -ftrace_ctx_switch(void *__rq, struct task_struct *prev, - struct task_struct *next) +static notrace void +wake_up_callback(void *probe_data, void *call_data, + const char *format, va_list *args) { - if (unlikely(atomic_read(&trace_record_cmdline_enabled))) - tracing_record_cmdline(prev); + struct task_struct *curr; + struct task_struct *task; + struct rq *__rq; - /* - * If tracer_switch_func only points to the local - * switch func, it still needs the ptr passed to it. - */ - ctx_switch_func(__rq, prev, next); + if (likely(!tracer_enabled)) + return; - /* - * Chain to the wakeup tracer (this is a NOP if disabled): - */ - wakeup_sched_switch(prev, next); -} + /* Skip pid %d state %ld */ + (void)va_arg(*args, int); + (void)va_arg(*args, long); + /* now get the meat: "rq %p task %p rq->curr %p" */ + __rq = va_arg(*args, typeof(__rq)); + task = va_arg(*args, typeof(task)); + curr = va_arg(*args, typeof(curr)); -void -ftrace_wake_up_task(void *__rq, struct task_struct *wakee, - struct task_struct *curr) -{ - wakeup_func(__rq, wakee, curr); + tracing_record_cmdline(task); + tracing_record_cmdline(curr); - /* - * Chain to the wakeup tracer (this is a NOP if disabled): - */ - wakeup_sched_wakeup(wakee, curr); + wakeup_func(probe_data, __rq, task, curr); } void @@ -132,15 +159,95 @@ static void sched_switch_reset(struct trace_array *tr) tracing_reset(tr->data[cpu]); } +static int tracing_sched_register(void) +{ + int ret; + + ret = marker_probe_register("kernel_sched_wakeup", + "pid %d state %ld ## rq %p task %p rq->curr %p", + wake_up_callback, + &ctx_trace); + if (ret) { + pr_info("wakeup trace: Couldn't add marker" + " probe to kernel_sched_wakeup\n"); + return ret; + } + + ret = marker_probe_register("kernel_sched_wakeup_new", + "pid %d state %ld ## rq %p task %p rq->curr %p", + wake_up_callback, + &ctx_trace); + if (ret) { + pr_info("wakeup trace: Couldn't add marker" + " probe to kernel_sched_wakeup_new\n"); + goto fail_deprobe; + } + + ret = marker_probe_register("kernel_sched_schedule", + "prev_pid %d next_pid %d prev_state %ld " + "## rq %p prev %p next %p", + sched_switch_callback, + &ctx_trace); + if (ret) { + pr_info("sched trace: Couldn't add marker" + " probe to kernel_sched_schedule\n"); + goto fail_deprobe_wake_new; + } + + return ret; +fail_deprobe_wake_new: + marker_probe_unregister("kernel_sched_wakeup_new", + wake_up_callback, + &ctx_trace); +fail_deprobe: + marker_probe_unregister("kernel_sched_wakeup", + wake_up_callback, + &ctx_trace); + return ret; +} + +static void tracing_sched_unregister(void) +{ + marker_probe_unregister("kernel_sched_schedule", + sched_switch_callback, + &ctx_trace); + marker_probe_unregister("kernel_sched_wakeup_new", + wake_up_callback, + &ctx_trace); + marker_probe_unregister("kernel_sched_wakeup", + wake_up_callback, + &ctx_trace); +} + +void tracing_start_sched_switch(void) +{ + long ref; + + ref = atomic_inc_return(&sched_ref); + if (ref == 1) + tracing_sched_register(); +} + +void tracing_stop_sched_switch(void) +{ + long ref; + + ref = atomic_dec_and_test(&sched_ref); + if (ref) + tracing_sched_unregister(); +} + static void start_sched_trace(struct trace_array *tr) { sched_switch_reset(tr); atomic_inc(&trace_record_cmdline_enabled); tracer_enabled = 1; + tracing_start_sched_switch(); } static void stop_sched_trace(struct trace_array *tr) { + tracing_stop_sched_switch(); atomic_dec(&trace_record_cmdline_enabled); tracer_enabled = 0; } @@ -181,6 +288,14 @@ static struct tracer sched_switch_trace __read_mostly = __init static int init_sched_switch_trace(void) { + int ret = 0; + + if (atomic_read(&sched_ref)) + ret = tracing_sched_register(); + if (ret) { + pr_info("error registering scheduler trace\n"); + return ret; + } return register_tracer(&sched_switch_trace); } device_initcall(init_sched_switch_trace); diff --git a/kernel/trace/trace_sched_wakeup.c b/kernel/trace/trace_sched_wakeup.c index 5948011006bc..5d2fb48e47f8 100644 --- a/kernel/trace/trace_sched_wakeup.c +++ b/kernel/trace/trace_sched_wakeup.c @@ -15,6 +15,7 @@ #include <linux/kallsyms.h> #include <linux/uaccess.h> #include <linux/ftrace.h> +#include <linux/marker.h> #include "trace.h" @@ -44,11 +45,13 @@ static int report_latency(cycle_t delta) return 1; } -void -wakeup_sched_switch(struct task_struct *prev, struct task_struct *next) +static void notrace +wakeup_sched_switch(void *private, void *rq, struct task_struct *prev, + struct task_struct *next) { unsigned long latency = 0, t0 = 0, t1 = 0; - struct trace_array *tr = wakeup_trace; + struct trace_array **ptr = private; + struct trace_array *tr = *ptr; struct trace_array_cpu *data; cycle_t T0, T1, delta; unsigned long flags; @@ -113,6 +116,31 @@ out: atomic_dec(&tr->data[cpu]->disabled); } +static notrace void +sched_switch_callback(void *probe_data, void *call_data, + const char *format, va_list *args) +{ + struct task_struct *prev; + struct task_struct *next; + struct rq *__rq; + + /* skip prev_pid %d next_pid %d prev_state %ld */ + (void)va_arg(*args, int); + (void)va_arg(*args, int); + (void)va_arg(*args, long); + __rq = va_arg(*args, typeof(__rq)); + prev = va_arg(*args, typeof(prev)); + next = va_arg(*args, typeof(next)); + + tracing_record_cmdline(prev); + + /* + * If tracer_switch_func only points to the local + * switch func, it still needs the ptr passed to it. + */ + wakeup_sched_switch(probe_data, __rq, prev, next); +} + static void __wakeup_reset(struct trace_array *tr) { struct trace_array_cpu *data; @@ -188,19 +216,68 @@ out: atomic_dec(&tr->data[cpu]->disabled); } -void wakeup_sched_wakeup(struct task_struct *wakee, struct task_struct *curr) +static notrace void +wake_up_callback(void *probe_data, void *call_data, + const char *format, va_list *args) { + struct trace_array **ptr = probe_data; + struct trace_array *tr = *ptr; + struct task_struct *curr; + struct task_struct *task; + struct rq *__rq; + if (likely(!tracer_enabled)) return; + /* Skip pid %d state %ld */ + (void)va_arg(*args, int); + (void)va_arg(*args, long); + /* now get the meat: "rq %p task %p rq->curr %p" */ + __rq = va_arg(*args, typeof(__rq)); + task = va_arg(*args, typeof(task)); + curr = va_arg(*args, typeof(curr)); + + tracing_record_cmdline(task); tracing_record_cmdline(curr); - tracing_record_cmdline(wakee); - wakeup_check_start(wakeup_trace, wakee, curr); + wakeup_check_start(tr, task, curr); } static void start_wakeup_tracer(struct trace_array *tr) { + int ret; + + ret = marker_probe_register("kernel_sched_wakeup", + "pid %d state %ld ## rq %p task %p rq->curr %p", + wake_up_callback, + &wakeup_trace); + if (ret) { + pr_info("wakeup trace: Couldn't add marker" + " probe to kernel_sched_wakeup\n"); + return; + } + + ret = marker_probe_register("kernel_sched_wakeup_new", + "pid %d state %ld ## rq %p task %p rq->curr %p", + wake_up_callback, + &wakeup_trace); + if (ret) { + pr_info("wakeup trace: Couldn't add marker" + " probe to kernel_sched_wakeup_new\n"); + goto fail_deprobe; + } + + ret = marker_probe_register("kernel_sched_schedule", + "prev_pid %d next_pid %d prev_state %ld " + "## rq %p prev %p next %p", + sched_switch_callback, + &wakeup_trace); + if (ret) { + pr_info("sched trace: Couldn't add marker" + " probe to kernel_sched_schedule\n"); + goto fail_deprobe_wake_new; + } + wakeup_reset(tr); /* @@ -215,11 +292,28 @@ static void start_wakeup_tracer(struct trace_array *tr) tracer_enabled = 1; return; +fail_deprobe_wake_new: + marker_probe_unregister("kernel_sched_wakeup_new", + wake_up_callback, + &wakeup_trace); +fail_deprobe: + marker_probe_unregister("kernel_sched_wakeup", + wake_up_callback, + &wakeup_trace); } static void stop_wakeup_tracer(struct trace_array *tr) { tracer_enabled = 0; + marker_probe_unregister("kernel_sched_schedule", + sched_switch_callback, + &wakeup_trace); + marker_probe_unregister("kernel_sched_wakeup_new", + wake_up_callback, + &wakeup_trace); + marker_probe_unregister("kernel_sched_wakeup", + wake_up_callback, + &wakeup_trace); } static void wakeup_tracer_init(struct trace_array *tr) diff --git a/kernel/workqueue.c b/kernel/workqueue.c index 29fc39f1029c..28c2b2c96ac7 100644 --- a/kernel/workqueue.c +++ b/kernel/workqueue.c @@ -397,7 +397,7 @@ void flush_workqueue(struct workqueue_struct *wq) might_sleep(); lock_acquire(&wq->lockdep_map, 0, 0, 0, 2, _THIS_IP_); lock_release(&wq->lockdep_map, 1, _THIS_IP_); - for_each_cpu_mask(cpu, *cpu_map) + for_each_cpu_mask_nr(cpu, *cpu_map) flush_cpu_workqueue(per_cpu_ptr(wq->cpu_wq, cpu)); } EXPORT_SYMBOL_GPL(flush_workqueue); @@ -477,7 +477,7 @@ static void wait_on_work(struct work_struct *work) wq = cwq->wq; cpu_map = wq_cpu_map(wq); - for_each_cpu_mask(cpu, *cpu_map) + for_each_cpu_mask_nr(cpu, *cpu_map) wait_on_cpu_work(per_cpu_ptr(wq->cpu_wq, cpu), work); } @@ -813,7 +813,7 @@ void destroy_workqueue(struct workqueue_struct *wq) list_del(&wq->list); spin_unlock(&workqueue_lock); - for_each_cpu_mask(cpu, *cpu_map) + for_each_cpu_mask_nr(cpu, *cpu_map) cleanup_workqueue_thread(per_cpu_ptr(wq->cpu_wq, cpu)); put_online_cpus(); diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 65001871bf00..85187ff2ea2a 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -147,7 +147,7 @@ config DETECT_SOFTLOCKUP help Say Y here to enable the kernel to detect "soft lockups", which are bugs that cause the kernel to loop in kernel - mode for more than 10 seconds, without giving other tasks a + mode for more than 60 seconds, without giving other tasks a chance to run. When a soft-lockup is detected, the kernel will print the @@ -159,6 +159,30 @@ config DETECT_SOFTLOCKUP can be detected via the NMI-watchdog, on platforms that support it.) +config BOOTPARAM_SOFTLOCKUP_PANIC + bool "Panic (Reboot) On Soft Lockups" + depends on DETECT_SOFTLOCKUP + help + Say Y here to enable the kernel to panic on "soft lockups", + which are bugs that cause the kernel to loop in kernel + mode for more than 60 seconds, without giving other tasks a + chance to run. + + The panic can be used in combination with panic_timeout, + to cause the system to reboot automatically after a + lockup has been detected. This feature is useful for + high-availability systems that have uptime guarantees and + where a lockup must be resolved ASAP. + + Say N if unsure. + +config BOOTPARAM_SOFTLOCKUP_PANIC_VALUE + int + depends on DETECT_SOFTLOCKUP + range 0 1 + default 0 if !BOOTPARAM_SOFTLOCKUP_PANIC + default 1 if BOOTPARAM_SOFTLOCKUP_PANIC + config SCHED_DEBUG bool "Collect scheduler debugging info" depends on DEBUG_KERNEL && PROC_FS diff --git a/lib/cpumask.c b/lib/cpumask.c index bb4f76d3c3e7..5f97dc25ef9c 100644 --- a/lib/cpumask.c +++ b/lib/cpumask.c @@ -15,6 +15,15 @@ int __next_cpu(int n, const cpumask_t *srcp) } EXPORT_SYMBOL(__next_cpu); +#if NR_CPUS > 64 +int __next_cpu_nr(int n, const cpumask_t *srcp) +{ + return min_t(int, nr_cpu_ids, + find_next_bit(srcp->bits, nr_cpu_ids, n+1)); +} +EXPORT_SYMBOL(__next_cpu_nr); +#endif + int __any_online_cpu(const cpumask_t *mask) { int cpu; diff --git a/lib/textsearch.c b/lib/textsearch.c index be8bda3862f5..a3e500ad51d7 100644 --- a/lib/textsearch.c +++ b/lib/textsearch.c @@ -97,6 +97,7 @@ #include <linux/types.h> #include <linux/string.h> #include <linux/init.h> +#include <linux/rculist.h> #include <linux/rcupdate.h> #include <linux/err.h> #include <linux/textsearch.h> diff --git a/localversion-sched-devel.git b/localversion-sched-devel.git new file mode 100644 index 000000000000..4060b7ac0145 --- /dev/null +++ b/localversion-sched-devel.git @@ -0,0 +1 @@ +-sched-devel.git diff --git a/mm/allocpercpu.c b/mm/allocpercpu.c index f4026bae6eed..b5411c2c47d4 100644 --- a/mm/allocpercpu.c +++ b/mm/allocpercpu.c @@ -35,7 +35,7 @@ EXPORT_SYMBOL_GPL(percpu_depopulate); void __percpu_depopulate_mask(void *__pdata, cpumask_t *mask) { int cpu; - for_each_cpu_mask(cpu, *mask) + for_each_cpu_mask_nr(cpu, *mask) percpu_depopulate(__pdata, cpu); } EXPORT_SYMBOL_GPL(__percpu_depopulate_mask); @@ -86,7 +86,7 @@ int __percpu_populate_mask(void *__pdata, size_t size, gfp_t gfp, int cpu; cpus_clear(populated); - for_each_cpu_mask(cpu, *mask) + for_each_cpu_mask_nr(cpu, *mask) if (unlikely(!percpu_populate(__pdata, size, gfp, cpu))) { __percpu_depopulate_mask(__pdata, &populated); return -ENOMEM; diff --git a/mm/filemap.c b/mm/filemap.c index 239d36163bbe..4c0502fb1f62 100644 --- a/mm/filemap.c +++ b/mm/filemap.c @@ -33,6 +33,7 @@ #include <linux/cpuset.h> #include <linux/hardirq.h> /* for BUG_ON(!in_atomic()) only */ #include <linux/memcontrol.h> +#include <linux/marker.h> #include "internal.h" /* @@ -540,9 +541,15 @@ void wait_on_page_bit(struct page *page, int bit_nr) { DEFINE_WAIT_BIT(wait, &page->flags, bit_nr); + trace_mark(mm_filemap_wait_start, "pfn %lu bit_nr %d", + page_to_pfn(page), bit_nr); + if (test_bit(bit_nr, &page->flags)) __wait_on_bit(page_waitqueue(page), &wait, sync_page, TASK_UNINTERRUPTIBLE); + + trace_mark(mm_filemap_wait_end, "pfn %lu bit_nr %d", + page_to_pfn(page), bit_nr); } EXPORT_SYMBOL(wait_on_page_bit); diff --git a/mm/hugetlb.c b/mm/hugetlb.c index bbf953eeb58b..d8f9cfb9582d 100644 --- a/mm/hugetlb.c +++ b/mm/hugetlb.c @@ -14,6 +14,7 @@ #include <linux/mempolicy.h> #include <linux/cpuset.h> #include <linux/mutex.h> +#include <linux/marker.h> #include <asm/page.h> #include <asm/pgtable.h> @@ -141,6 +142,7 @@ static void free_huge_page(struct page *page) int nid = page_to_nid(page); struct address_space *mapping; + trace_mark(mm_huge_page_free, "pfn %lu", page_to_pfn(page)); mapping = (struct address_space *) page_private(page); set_page_private(page, 0); BUG_ON(page_count(page)); @@ -509,6 +511,7 @@ static struct page *alloc_huge_page(struct vm_area_struct *vma, if (!IS_ERR(page)) { set_page_refcounted(page); set_page_private(page, (unsigned long) mapping); + trace_mark(mm_huge_page_alloc, "pfn %lu", page_to_pfn(page)); } return page; } diff --git a/mm/memory.c b/mm/memory.c index 48c122d42ed7..ca85a6ce3ffa 100644 --- a/mm/memory.c +++ b/mm/memory.c @@ -45,12 +45,15 @@ #include <linux/swap.h> #include <linux/highmem.h> #include <linux/pagemap.h> +#include <linux/marker.h> #include <linux/rmap.h> #include <linux/module.h> #include <linux/delayacct.h> #include <linux/init.h> #include <linux/writeback.h> #include <linux/memcontrol.h> +#include <linux/kprobes.h> +#include <linux/mutex.h> #include <asm/pgalloc.h> #include <asm/uaccess.h> @@ -96,6 +99,12 @@ int randomize_va_space __read_mostly = 2; #endif +/* + * mutex protecting text section modification (dynamic code patching). + * some users need to sleep (allocating memory...) while they hold this lock. + */ +static DEFINE_MUTEX(text_mutex); + static int __init disable_randmaps(char *s) { randomize_va_space = 0; @@ -2131,6 +2140,12 @@ static int do_swap_page(struct mm_struct *mm, struct vm_area_struct *vma, /* Had to read the page from swap area: Major fault */ ret = VM_FAULT_MAJOR; count_vm_event(PGMAJFAULT); +#ifdef CONFIG_SWAP + trace_mark(mm_swap_in, "pfn %lu filp %p offset %lu", + page_to_pfn(page), + get_swap_info_struct(swp_type(entry))->swap_file, + swp_offset(entry)); +#endif } if (mem_cgroup_charge(page, mm, GFP_KERNEL)) { @@ -2582,30 +2597,46 @@ unlock: int handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma, unsigned long address, int write_access) { + int res; pgd_t *pgd; pud_t *pud; pmd_t *pmd; pte_t *pte; + trace_mark(mm_handle_fault_entry, + "address %lu ip #p%ld write_access %d", + address, KSTK_EIP(current), write_access); + __set_current_state(TASK_RUNNING); count_vm_event(PGFAULT); - if (unlikely(is_vm_hugetlb_page(vma))) - return hugetlb_fault(mm, vma, address, write_access); + if (unlikely(is_vm_hugetlb_page(vma))) { + res = hugetlb_fault(mm, vma, address, write_access); + goto end; + } pgd = pgd_offset(mm, address); pud = pud_alloc(mm, pgd, address); - if (!pud) - return VM_FAULT_OOM; + if (!pud) { + res = VM_FAULT_OOM; + goto end; + } pmd = pmd_alloc(mm, pud, address); - if (!pmd) - return VM_FAULT_OOM; + if (!pmd) { + res = VM_FAULT_OOM; + goto end; + } pte = pte_alloc_map(mm, pmd, address); - if (!pte) - return VM_FAULT_OOM; + if (!pte) { + res = VM_FAULT_OOM; + goto end; + } - return handle_pte_fault(mm, vma, address, pte, pmd, write_access); + res = handle_pte_fault(mm, vma, address, pte, pmd, write_access); +end: + trace_mark(mm_handle_fault_exit, MARK_NOARGS); + return res; } #ifndef __PAGETABLE_PUD_FOLDED @@ -2810,3 +2841,29 @@ void print_vma_addr(char *prefix, unsigned long ip) } up_read(¤t->mm->mmap_sem); } + +/** + * kernel_text_lock - Take the kernel text modification lock + * + * Insures mutual write exclusion of kernel and modules text live text + * modification. Should be used for code patching. + * Users of this lock can sleep. + */ +void __kprobes kernel_text_lock(void) +{ + mutex_lock(&text_mutex); +} +EXPORT_SYMBOL_GPL(kernel_text_lock); + +/** + * kernel_text_unlock - Release the kernel text modification lock + * + * Insures mutual write exclusion of kernel and modules text live text + * modification. Should be used for code patching. + * Users of this lock can sleep. + */ +void __kprobes kernel_text_unlock(void) +{ + mutex_unlock(&text_mutex); +} +EXPORT_SYMBOL_GPL(kernel_text_unlock); diff --git a/mm/page-writeback.c b/mm/page-writeback.c index 789b6adbef37..b38f700825fc 100644 --- a/mm/page-writeback.c +++ b/mm/page-writeback.c @@ -126,8 +126,6 @@ static void background_writeout(unsigned long _min_pages); static struct prop_descriptor vm_completions; static struct prop_descriptor vm_dirties; -static unsigned long determine_dirtyable_memory(void); - /* * couple the period to the dirty_ratio: * @@ -347,7 +345,13 @@ static unsigned long highmem_dirtyable_memory(unsigned long total) #endif } -static unsigned long determine_dirtyable_memory(void) +/** + * determine_dirtyable_memory - amount of memory that may be used + * + * Returns the numebr of pages that can currently be freed and used + * by the kernel for direct mappings. + */ +unsigned long determine_dirtyable_memory(void) { unsigned long x; diff --git a/mm/page_alloc.c b/mm/page_alloc.c index bdd5c432c426..2bbe7208a441 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -46,6 +46,7 @@ #include <linux/page-isolation.h> #include <linux/memcontrol.h> #include <linux/debugobjects.h> +#include <linux/marker.h> #include <asm/tlbflush.h> #include <asm/div64.h> @@ -528,6 +529,9 @@ static void __free_pages_ok(struct page *page, unsigned int order) int i; int reserved = 0; + trace_mark(mm_page_free, "order %u pfn %lu", + order, page_to_pfn(page)); + for (i = 0 ; i < (1 << order) ; ++i) reserved += free_pages_check(page + i); if (reserved) @@ -994,6 +998,8 @@ static void free_hot_cold_page(struct page *page, int cold) struct per_cpu_pages *pcp; unsigned long flags; + trace_mark(mm_page_free, "order %u pfn %lu", 0, page_to_pfn(page)); + if (PageAnon(page)) page->mapping = NULL; if (free_pages_check(page)) @@ -1655,6 +1661,9 @@ nopage: show_mem(); } got_pg: + if (page) + trace_mark(mm_page_alloc, "order %u pfn %lu", order, + page_to_pfn(page)); return page; } diff --git a/mm/page_io.c b/mm/page_io.c index 065c4480eaf0..29555852b280 100644 --- a/mm/page_io.c +++ b/mm/page_io.c @@ -17,6 +17,7 @@ #include <linux/bio.h> #include <linux/swapops.h> #include <linux/writeback.h> +#include <linux/marker.h> #include <asm/pgtable.h> static struct bio *get_swap_bio(gfp_t gfp_flags, pgoff_t index, @@ -114,6 +115,11 @@ int swap_writepage(struct page *page, struct writeback_control *wbc) rw |= (1 << BIO_RW_SYNC); count_vm_event(PSWPOUT); set_page_writeback(page); + trace_mark(mm_swap_out, "pfn %lu filp %p offset %lu", + page_to_pfn(page), + get_swap_info_struct(swp_type( + page_swp_entry(page)))->swap_file, + swp_offset(page_swp_entry(page))); unlock_page(page); submit_bio(rw, bio); out: diff --git a/mm/swapfile.c b/mm/swapfile.c index bd1bb5920306..6a648a3dcaad 100644 --- a/mm/swapfile.c +++ b/mm/swapfile.c @@ -28,6 +28,7 @@ #include <linux/capability.h> #include <linux/syscalls.h> #include <linux/memcontrol.h> +#include <linux/marker.h> #include <asm/pgtable.h> #include <asm/tlbflush.h> @@ -1310,6 +1311,7 @@ asmlinkage long sys_swapoff(const char __user * specialfile) swap_map = p->swap_map; p->swap_map = NULL; p->flags = 0; + trace_mark(mm_swap_file_close, "filp %p", swap_file); spin_unlock(&swap_lock); mutex_unlock(&swapon_mutex); vfree(swap_map); @@ -1695,6 +1697,8 @@ asmlinkage long sys_swapon(const char __user * specialfile, int swap_flags) } else { swap_info[prev].next = p - swap_info; } + trace_mark(mm_swap_file_open, "filp %p filename %s", + swap_file, name); spin_unlock(&swap_lock); mutex_unlock(&swapon_mutex); error = 0; @@ -1848,3 +1852,22 @@ int valid_swaphandles(swp_entry_t entry, unsigned long *offset) *offset = ++toff; return nr_pages? ++nr_pages: 0; } + +void ltt_dump_swap_files(void *call_data) +{ + int type; + struct swap_info_struct *p = NULL; + + mutex_lock(&swapon_mutex); + for (type = swap_list.head; type >= 0; type = swap_info[type].next) { + p = swap_info + type; + if ((p->flags & SWP_ACTIVE) != SWP_ACTIVE) + continue; + __trace_mark(0, statedump_swap_files, call_data, + "filp %p vfsmount %p dname %s", + p->swap_file, p->swap_file->f_vfsmnt, + p->swap_file->f_dentry->d_name.name); + } + mutex_unlock(&swapon_mutex); +} +EXPORT_SYMBOL_GPL(ltt_dump_swap_files); diff --git a/mm/vmstat.c b/mm/vmstat.c index 1a32130b958c..67399b517e3e 100644 --- a/mm/vmstat.c +++ b/mm/vmstat.c @@ -26,7 +26,7 @@ static void sum_vm_events(unsigned long *ret, cpumask_t *cpumask) memset(ret, 0, NR_VM_EVENT_ITEMS * sizeof(unsigned long)); - for_each_cpu_mask(cpu, *cpumask) { + for_each_cpu_mask_nr(cpu, *cpumask) { struct vm_event_state *this = &per_cpu(vm_event_states, cpu); for (i = 0; i < NR_VM_EVENT_ITEMS; i++) diff --git a/net/802/psnap.c b/net/802/psnap.c index 31128cb92a23..ea4643931446 100644 --- a/net/802/psnap.c +++ b/net/802/psnap.c @@ -20,6 +20,7 @@ #include <linux/mm.h> #include <linux/in.h> #include <linux/init.h> +#include <linux/rculist.h> static LIST_HEAD(snap_list); static DEFINE_SPINLOCK(snap_lock); diff --git a/net/8021q/vlan.c b/net/8021q/vlan.c index 2a739adaa92b..e7ddbfa0e02f 100644 --- a/net/8021q/vlan.c +++ b/net/8021q/vlan.c @@ -27,6 +27,7 @@ #include <linux/mm.h> #include <linux/in.h> #include <linux/init.h> +#include <linux/rculist.h> #include <net/p8022.h> #include <net/arp.h> #include <linux/rtnetlink.h> diff --git a/net/bridge/br_fdb.c b/net/bridge/br_fdb.c index 72c5976a5ce3..142060f02054 100644 --- a/net/bridge/br_fdb.c +++ b/net/bridge/br_fdb.c @@ -15,6 +15,7 @@ #include <linux/kernel.h> #include <linux/init.h> +#include <linux/rculist.h> #include <linux/spinlock.h> #include <linux/times.h> #include <linux/netdevice.h> diff --git a/net/bridge/br_stp.c b/net/bridge/br_stp.c index e38034aa56f5..9e96ffcd29a3 100644 --- a/net/bridge/br_stp.c +++ b/net/bridge/br_stp.c @@ -13,6 +13,7 @@ * 2 of the License, or (at your option) any later version. */ #include <linux/kernel.h> +#include <linux/rculist.h> #include "br_private.h" #include "br_private_stp.h" diff --git a/net/core/dev.c b/net/core/dev.c index d334446a8eaf..2862fe1f2e00 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -119,6 +119,7 @@ #include <linux/err.h> #include <linux/ctype.h> #include <linux/if_arp.h> +#include <linux/marker.h> #include "net-sysfs.h" @@ -1643,6 +1644,8 @@ int dev_queue_xmit(struct sk_buff *skb) } gso: + trace_mark(net_dev_xmit, "skb %p protocol #2u%hu", skb, skb->protocol); + spin_lock_prefetch(&dev->queue_lock); /* Disable soft irqs for various locks below. Also @@ -2043,6 +2046,9 @@ int netif_receive_skb(struct sk_buff *skb) __get_cpu_var(netdev_rx_stat).total++; + trace_mark(net_dev_receive, "skb %p protocol #2u%hu", + skb, skb->protocol); + skb_reset_network_header(skb); skb_reset_transport_header(skb); skb->mac_len = skb->network_header - skb->mac_header; @@ -2231,7 +2237,7 @@ out: */ if (!cpus_empty(net_dma.channel_mask)) { int chan_idx; - for_each_cpu_mask(chan_idx, net_dma.channel_mask) { + for_each_cpu_mask_nr(chan_idx, net_dma.channel_mask) { struct dma_chan *chan = net_dma.channels[chan_idx]; if (chan) dma_async_memcpy_issue_pending(chan); @@ -4292,7 +4298,7 @@ static void net_dma_rebalance(struct net_dma *net_dma) i = 0; cpu = first_cpu(cpu_online_map); - for_each_cpu_mask(chan_idx, net_dma->channel_mask) { + for_each_cpu_mask_nr(chan_idx, net_dma->channel_mask) { chan = net_dma->channels[chan_idx]; n = ((num_online_cpus() / cpus_weight(net_dma->channel_mask)) diff --git a/net/ipv4/devinet.c b/net/ipv4/devinet.c index 6848e4760f34..1781f0bb388a 100644 --- a/net/ipv4/devinet.c +++ b/net/ipv4/devinet.c @@ -56,6 +56,7 @@ #include <linux/sysctl.h> #endif #include <linux/kmod.h> +#include <linux/marker.h> #include <net/arp.h> #include <net/ip.h> @@ -258,6 +259,8 @@ static void __inet_del_ifa(struct in_device *in_dev, struct in_ifaddr **ifap, struct in_ifaddr **ifap1 = &ifa1->ifa_next; while ((ifa = *ifap1) != NULL) { + trace_mark(net_del_ifa_ipv4, "label %s", + ifa->ifa_label); if (!(ifa->ifa_flags & IFA_F_SECONDARY) && ifa1->ifa_scope <= ifa->ifa_scope) last_prim = ifa; @@ -364,6 +367,9 @@ static int __inet_insert_ifa(struct in_ifaddr *ifa, struct nlmsghdr *nlh, } ifa->ifa_flags |= IFA_F_SECONDARY; } + trace_mark(net_insert_ifa_ipv4, "label %s address #4u%lu", + ifa->ifa_label, + (unsigned long)ifa->ifa_address); } if (!(ifa->ifa_flags & IFA_F_SECONDARY)) { diff --git a/net/iucv/iucv.c b/net/iucv/iucv.c index 918970762131..8de511070593 100644 --- a/net/iucv/iucv.c +++ b/net/iucv/iucv.c @@ -497,7 +497,7 @@ static void iucv_setmask_up(void) /* Disable all cpu but the first in cpu_irq_cpumask. */ cpumask = iucv_irq_cpumask; cpu_clear(first_cpu(iucv_irq_cpumask), cpumask); - for_each_cpu_mask(cpu, cpumask) + for_each_cpu_mask_nr(cpu, cpumask) smp_call_function_single(cpu, iucv_block_cpu, NULL, 0, 1); } diff --git a/net/netlabel/netlabel_domainhash.c b/net/netlabel/netlabel_domainhash.c index 02c2f7c0b255..643c032a3a57 100644 --- a/net/netlabel/netlabel_domainhash.c +++ b/net/netlabel/netlabel_domainhash.c @@ -30,8 +30,7 @@ */ #include <linux/types.h> -#include <linux/rcupdate.h> -#include <linux/list.h> +#include <linux/rculist.h> #include <linux/skbuff.h> #include <linux/spinlock.h> #include <linux/string.h> diff --git a/net/socket.c b/net/socket.c index 66c4a8cf6db9..bd5e9671bcbe 100644 --- a/net/socket.c +++ b/net/socket.c @@ -85,6 +85,7 @@ #include <linux/audit.h> #include <linux/wireless.h> #include <linux/nsproxy.h> +#include <linux/marker.h> #include <asm/uaccess.h> #include <asm/unistd.h> @@ -567,6 +568,11 @@ int sock_sendmsg(struct socket *sock, struct msghdr *msg, size_t size) struct sock_iocb siocb; int ret; + trace_mark(net_socket_sendmsg, + "sock %p family %d type %d protocol %d size %zu", + sock, sock->sk->sk_family, sock->sk->sk_type, + sock->sk->sk_protocol, size); + init_sync_kiocb(&iocb, NULL); iocb.private = &siocb; ret = __sock_sendmsg(&iocb, sock, msg, size); @@ -650,7 +656,13 @@ int sock_recvmsg(struct socket *sock, struct msghdr *msg, struct sock_iocb siocb; int ret; + trace_mark(net_socket_recvmsg, + "sock %p family %d type %d protocol %d size %zu", + sock, sock->sk->sk_family, sock->sk->sk_type, + sock->sk->sk_protocol, size); + init_sync_kiocb(&iocb, NULL); + iocb.private = &siocb; ret = __sock_recvmsg(&iocb, sock, msg, size, flags); if (-EIOCBQUEUED == ret) @@ -1226,6 +1238,11 @@ asmlinkage long sys_socket(int family, int type, int protocol) if (retval < 0) goto out_release; + trace_mark(net_socket_create, + "sock %p family %d type %d protocol %d fd %d", + sock, sock->sk->sk_family, sock->sk->sk_type, + sock->sk->sk_protocol, retval); + out: /* It may be already another descriptor 8) Not kernel problem. */ return retval; @@ -2024,6 +2041,8 @@ asmlinkage long sys_socketcall(int call, unsigned long __user *args) a0 = a[0]; a1 = a[1]; + trace_mark(net_socket_call, "call %d a0 %lu", call, a0); + switch (call) { case SYS_SOCKET: err = sys_socket(a0, a1, a[2]); diff --git a/net/sunrpc/svc.c b/net/sunrpc/svc.c index d74c2d269539..271e77771bee 100644 --- a/net/sunrpc/svc.c +++ b/net/sunrpc/svc.c @@ -604,7 +604,7 @@ __svc_create_thread(svc_thread_fn func, struct svc_serv *serv, error = kernel_thread((int (*)(void *)) func, rqstp, 0); if (have_oldmask) - set_cpus_allowed(current, oldmask); + set_cpus_allowed_ptr(current, &oldmask); if (error < 0) goto out_thread; diff --git a/tests/rcutorture.c b/tests/rcutorture.c index 33acc424667e..0334b6a8baca 100644 --- a/tests/rcutorture.c +++ b/tests/rcutorture.c @@ -192,6 +192,7 @@ struct rcu_torture_ops { int (*completed)(void); void (*deferredfree)(struct rcu_torture *p); void (*sync)(void); + void (*cb_barrier)(void); int (*stats)(char *page); char *name; }; @@ -265,6 +266,7 @@ static struct rcu_torture_ops rcu_ops = { .completed = rcu_torture_completed, .deferredfree = rcu_torture_deferred_free, .sync = synchronize_rcu, + .cb_barrier = rcu_barrier, .stats = NULL, .name = "rcu" }; @@ -304,6 +306,7 @@ static struct rcu_torture_ops rcu_sync_ops = { .completed = rcu_torture_completed, .deferredfree = rcu_sync_torture_deferred_free, .sync = synchronize_rcu, + .cb_barrier = NULL, .stats = NULL, .name = "rcu_sync" }; @@ -364,6 +367,7 @@ static struct rcu_torture_ops rcu_bh_ops = { .completed = rcu_bh_torture_completed, .deferredfree = rcu_bh_torture_deferred_free, .sync = rcu_bh_torture_synchronize, + .cb_barrier = rcu_barrier_bh, .stats = NULL, .name = "rcu_bh" }; @@ -377,6 +381,7 @@ static struct rcu_torture_ops rcu_bh_sync_ops = { .completed = rcu_bh_torture_completed, .deferredfree = rcu_sync_torture_deferred_free, .sync = rcu_bh_torture_synchronize, + .cb_barrier = NULL, .stats = NULL, .name = "rcu_bh_sync" }; @@ -458,6 +463,7 @@ static struct rcu_torture_ops srcu_ops = { .completed = srcu_torture_completed, .deferredfree = rcu_sync_torture_deferred_free, .sync = srcu_torture_synchronize, + .cb_barrier = NULL, .stats = srcu_torture_stats, .name = "srcu" }; @@ -482,6 +488,11 @@ static int sched_torture_completed(void) return 0; } +static void rcu_sched_torture_deferred_free(struct rcu_torture *p) +{ + call_rcu_sched(&p->rtort_rcu, rcu_torture_cb); +} + static void sched_torture_synchronize(void) { synchronize_sched(); @@ -494,12 +505,27 @@ static struct rcu_torture_ops sched_ops = { .readdelay = rcu_read_delay, /* just reuse rcu's version. */ .readunlock = sched_torture_read_unlock, .completed = sched_torture_completed, - .deferredfree = rcu_sync_torture_deferred_free, + .deferredfree = rcu_sched_torture_deferred_free, .sync = sched_torture_synchronize, + .cb_barrier = rcu_barrier_sched, .stats = NULL, .name = "sched" }; +static struct rcu_torture_ops sched_ops_sync = { + .init = rcu_sync_torture_init, + .cleanup = NULL, + .readlock = sched_torture_read_lock, + .readdelay = rcu_read_delay, /* just reuse rcu's version. */ + .readunlock = sched_torture_read_unlock, + .completed = sched_torture_completed, + .deferredfree = rcu_sync_torture_deferred_free, + .sync = sched_torture_synchronize, + .cb_barrier = NULL, + .stats = NULL, + .name = "sched_sync" +}; + /* * RCU torture writer kthread. Repeatedly substitutes a new structure * for that pointed to by rcu_torture_current, freeing the old structure @@ -848,7 +874,9 @@ rcu_torture_cleanup(void) stats_task = NULL; /* Wait for all RCU callbacks to fire. */ - rcu_barrier(); + + if (cur_ops->cb_barrier != NULL) + cur_ops->cb_barrier(); rcu_torture_stats_print(); /* -After- the stats thread is stopped! */ @@ -868,7 +896,7 @@ rcu_torture_init(void) int firsterr = 0; static struct rcu_torture_ops *torture_ops[] = { &rcu_ops, &rcu_sync_ops, &rcu_bh_ops, &rcu_bh_sync_ops, - &srcu_ops, &sched_ops, }; + &srcu_ops, &sched_ops, &sched_ops_sync, }; /* Process args and tell the world that the torturer is on the job. */ for (i = 0; i < ARRAY_SIZE(torture_ops); i++) { |