MezzanineEngine 
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
framescheduler.h
Go to the documentation of this file.
1 // The DAGFrameScheduler is a Multi-Threaded lock free and wait free scheduling library.
2 // © Copyright 2010 - 2014 BlackTopp Studios Inc.
3 /* This file is part of The DAGFrameScheduler.
4 
5  The DAGFrameScheduler is free software: you can redistribute it and/or modify
6  it under the terms of the GNU General Public License as published by
7  the Free Software Foundation, either version 3 of the License, or
8  (at your option) any later version.
9 
10  The DAGFrameScheduler is distributed in the hope that it will be useful,
11  but WITHOUT ANY WARRANTY; without even the implied warranty of
12  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13  GNU General Public License for more details.
14 
15  You should have received a copy of the GNU General Public License
16  along with The DAGFrameScheduler. If not, see <http://www.gnu.org/licenses/>.
17 */
18 /* The original authors have included a copy of the license specified above in the
19  'doc' folder. See 'gpl.txt'
20 */
21 /* We welcome the use of the DAGFrameScheduler to anyone, including companies who wish to
22  Build professional software and charge for their product.
23 
24  However there are some practical restrictions, so if your project involves
25  any of the following you should contact us and we will try to work something
26  out:
27  - DRM or Copy Protection of any kind(except Copyrights)
28  - Software Patents You Do Not Wish to Freely License
29  - Any Kind of Linking to Non-GPL licensed Works
30  - Are Currently In Violation of Another Copyright Holder's GPL License
31  - If You want to change our code and not add a few hundred MB of stuff to
32  your distribution
33 
34  These and other limitations could cause serious legal problems if you ignore
35  them, so it is best to simply contact us or the Free Software Foundation, if
36  you have any questions.
37 
38  Joseph Toppi - toppij@gmail.com
39  John Blackwood - makoenergy02@gmail.com
40 */
41 #ifndef _framescheduler_h
42 #define _framescheduler_h
43 
44 #include "datatypes.h"
45 
46 #if !defined(SWIG) || defined(SWIG_THREADING) // Do not read when in swig and not in the threading module
47 #include "doublebufferedresource.h"
48 #include "thread.h"
49 #include "workunitkey.h"
50 #include "spinlock.h"
51 #include "systemcalls.h"
52 #include "rollingaverage.h"
53 #endif
54 
55 #ifdef MEZZ_USEBARRIERSEACHFRAME
56  #include "barrier.h"
57 #endif
58 
59 /// @file
60 /// @brief This file has the Declarations for the main FrameScheduler class.
61 
62 namespace Mezzanine
63 {
64  namespace Threading
65  {
66  class MonopolyWorkUnit;
67  class iWorkUnit;
68  class LogAggregator;
69  class FrameScheduler;
70  class WorkSorter;
71 
72  /// @brief This is central object in this algorithm, it is responsible for spawning threads and managing the order that work units are executed.
73  /// @details For a detailed description of the @ref algorithm_sec "Algorithm" this implements see the @ref algorithm_sec "Algorithm" section on
74  /// the Main page.
76  {
77  friend class LogAggregator;
78  friend class WorkSorter;
79 
80  protected:
81  ////////////////////////////////////////////////////////////////////////////////
82  // Data Members
83 
84  /// @brief A collection of all the work units that are not Monopolies and do not have affinity for a given thread.
85  /// @details This stores a sorted listing(currently a vector) of @ref Mezzanine::Threading::WorkUnitKey "WorkUnitKey" instances.
86  /// These include just the metadata required for sorting @ref Mezzanine::Threading::iWorkUnit "iWorkUnit"s. Higher priority
87  /// @ref Mezzanine::Threading::iWorkUnit "iWorkUnit"s are higher/later in the collection. This list is sorted by calls
88  /// to @ref Mezzanine::Threading::FrameScheduler::SortWorkUnitsMain "SortWorkUnitsMain" or @ref Mezzanine::Threading::FrameScheduler::SortWorkUnitsAll "SortWorkUnitsAll".
89  std::vector<WorkUnitKey> WorkUnitsMain;
90 
91  /// @brief An iterator suitable for iterating over the main pool of work units.
92  typedef std::vector<WorkUnitKey>::iterator IteratorMain;
93 
94  /// @brief A const iterator suitable for iterating over the main pool of work units.
95  typedef std::vector<WorkUnitKey>::const_iterator ConstIteratorMain;
96 
97  /// @brief A collection of @ref Mezzanine::Threading::iWorkUnit "iWorkUnit"s that must be run on the main thread.
98  /// @details This is very similar to @ref WorkUnitsMain except that the @ref Mezzanine::Threading::iWorkUnit "iWorkUnit"s
99  /// are only run in the main thread and are sorted by calls to @ref SortWorkUnitsAll or @ref SortWorkUnitsAffinity .
100  std::vector<WorkUnitKey> WorkUnitsAffinity;
101 
102  /// @brief An iterator suitable for iterating over the main pool of work units.
103  typedef std::vector<WorkUnitKey>::iterator IteratorAffinity;
104 
105  /// @brief A const iterator suitable for iterating over the main pool of work units.
106  typedef std::vector<WorkUnitKey>::const_iterator ConstIteratorAffinity;
107 
108  /// @brief A structure designed to minimalistically represent Dependency and Reverse Dependency Graphs in work units.
109  typedef std::map<
110  iWorkUnit*,
111  std::set<iWorkUnit*>
113 
114  /// @brief This structure allows reverse lookup of dependencies.
115  /// @details This is is a key part of the workunit sorting algorithm. This is calculated during calls to generate
116  /// @ref Mezzanine::Threading::WorkUnitKey "WorkUnitKey"s,
117  /// @warning
118  /// @todo write this warning, it is important, but not easy to lay out.
120 
121  public:
122  /// @brief The kind of Resource the frame scheduler will use
124 
125  protected:
126  /// @brief This maintains ownership of all the thread specific resources.
127  /// @note There should be the same amount or more of these than entries in the Threads vector.
128  std::vector<Resource*> Resources;
129 
130  /// @brief A way to track an arbitrary number of threads.
131  /// @note There should never be more of these than Resources, and if there are more at the beginning of a frame the resources will be created in CreateThreads().
132  std::vector<Thread*> Threads;
133 
134  /// @brief A collection of all the monopolies this scheduler must run and keep ownership of.
135  std::vector<MonopolyWorkUnit*> WorkUnitsMonopolies;
136 
137  /// @brief An iterator suitable for iterating over the main pool of work units.
138  typedef std::vector<MonopolyWorkUnit*>::iterator IteratorMonoply;
139 
140  /// @brief A const iterator suitable for iterating over the main pool of work units.
141  typedef std::vector<MonopolyWorkUnit*>::const_iterator ConstIteratorMonopoly;
142 
143  /// @brief A rolling average of Frame times.
145 
146  /// @brief A rolling average of Frame Pause times.
148 
149  /// @brief What time did the current Frame Start at.
151 
152  /// @brief What time did the current Frame Start at.
154 
155  /// @brief When the logs are aggregated, this is where they are sent
156  std::ostream* LogDestination;
157 
158  /// @brief If this pointer is non-zero then the @ref WorkSorter it points at will be used to sort WorkUnits.
160 
161  #ifdef MEZZ_USEBARRIERSEACHFRAME
162  public:
163  /// @brief Used to synchronize the starting an stopping of all threads before the frame starts.
164  Barrier StartFrameSync;
165 
166  /// @brief Used to synchronize the starting and stopping of all threads after work is done before the frame ends.
167  Barrier EndFrameSync;
168 
169  /// @brief When using barriers instead of thread creation for synchronization this is what tells the threads to end.
170  Int32 LastFrame;
171  protected:
172  #endif
173 
174  /// @brief Protects DoubleBufferedResources during creation from being accessed by the LogAggregator.
176 
177  #ifdef MEZZ_USEATOMICSTODECACHECOMPLETEWORK
178  /// @brief Indicates the beginning of work that must be searched when starting a fresh search for work in WorkUnitsMain.
179  Int32 DecacheMain;
180 
181  /// @brief Indicates the beginning of work that must be searched when starting a fresh search for work in WorkUnitsAffinity.
182  Int32 DecacheAffinity;
183  #endif
184 
185  /// @brief How many threads will this try to execute with in the next frame.
187 
188  /// @brief Used to store a count of frames from the begining of game execution.
189  /// @warning At 60 Frames per second this loops in 2 years, 3 months, 6 days and around 18 hours, this may not be suitable for high uptime servers. Using a MaxInt fixes this.
191 
192  /// @brief The Maximum frame rate this algorithm should run at.
194 
195  /// @brief To prevent frame time drift this many microseconds is subtracted from the wait period to allow time for calculations.
197 
198  /// @brief For some task it is important to know the ID of the main thread.
200 
201  /// @brief Set based on which constructor is called, and only used during destruction.
203 
204  /// @brief Brief Do we have to log ependencies have they changed since last logged?
205  /// @details Since each workunit tracks its own dependencies this cannot easily be directly checked. There
206  /// is a function to set and since dependencies are likely to be between adding a working and runninf a
207  /// frame, adding a workunit sets this flag.
209 
210  ////////////////////////////////////////////////////////////////////////////////
211  // Protected Methods
212 
213  /// @brief Used in destruction to tear down threads.
214  void CleanUpThreads();
215 
216  /// @brief Simply iterates over and deletes everything in Threads.
217  void DeleteThreads();
218 
219  /// @brief Adds the dependencies of the @ref Mezzanine::Threading::iWorkUnit "iWorkUnit"s in the passed containter to the the internal reverse dependency graph.
220  /// @param Units The container to examine for dependency relationships.
221  void UpdateDependentGraph(const std::vector<WorkUnitKey> &Units);
222 
223  /// @brief Iterate over the passed container of @ref WorkUnitKey "WorkUnitKey"s and refresh them with the correct data from their respective @ref Mezzanine::Threading::iWorkUnit "iWorkUnit"s
224  /// @param Units The container to examine for @ref WorkUnitKey "WorkUnitKey" metadata.
225  void UpdateWorkUnitKeys(std::vector<WorkUnitKey> &Units);
226 
227  public:
228 
229  ////////////////////////////////////////////////////////////////////////////////
230  // Construction and Destruction
231 
232  /// @brief Create a Framescheduler that owns a filestream for logging.
233  /// @param _LogDestination An fstream that will be closed and deleted when this framescheduler is destroyed. Defaults to a new Filestream Logging to local file.
234  /// @param StartingThreadCount How many threads. Defaults to the value returned by @ref Mezzanine::GetCPUCount "GetCPUCount()".
235  /// @warning This must be constructed from the Main(only) thread for any features with thread affinity to work correctly.
237  std::fstream* _LogDestination = 0,
238  Whole StartingThreadCount = GetCPUCount()
239  );
240 
241  /// @brief Create a Framescheduler, that logs to an unowned stream.
242  /// @param _LogDestination Any stream, other than an fstream, and it will be closed (not deleted) when this frame scheduler is destroyed.
243  /// @param StartingThreadCount How many threads. Defaults to the value returned by @ref Mezzanine::GetCPUCount "GetCPUCount()".
244  /// @warning This must be constructed from the Main(only) thread for any features with thread affinity to work correctly.
246  std::ostream* _LogDestination,
247  Whole StartingThreadCount = GetCPUCount()
248  );
249 
250  /// @brief Destructor
251  /// @details Deletes all std::fstream, WorkUnit, MonopolyWorkUnit and ThreadSpecificStorage objects that this was passed or created during its lifetime.
252  virtual ~FrameScheduler();
253 
254  ////////////////////////////////////////////////////////////////////////////////
255  // WorkUnit management
256 
257  /// @brief Add a normal Mezzanine::Threading::iWorkUnit to this For fcheduling.
258  /// @param MoreWork A pointer the the WorkUnit, that the FrameScheduler will take ownership of, and schedule for work.
259  /// @param WorkUnitName A name to uniquely identify this work unit in the logs
260  virtual void AddWorkUnitMain(iWorkUnit* MoreWork, const String& WorkUnitName);
261 
262  /// @brief Add a normal Mezzanine::Threading::iWorkUnit to this For scheduling.
263  /// @param MoreWork A pointer the the WorkUnit, that the FrameScheduler will take ownership of, and schedule for work.
264  /// @param WorkUnitName A name to uniquely identify this work unit in the logs
265  virtual void AddWorkUnitAffinity(iWorkUnit* MoreWork, const String& WorkUnitName);
266 
267  /// @brief Add a @ref MonopolyWorkUnit for execution at the beginning of the frame.
268  /// @param MoreWork A pointer to the @ref MonopolyWorkUnit to add.
269  /// @param WorkUnitName A name to uniquely identify this work unit in the logs
270  virtual void AddWorkUnitMonopoly(MonopolyWorkUnit* MoreWork, const String& WorkUnitName);
271 
272  /// @brief Sort the the main pool of WorkUnits to allow them to be used more efficiently in the next frame executed.
273  /// @param UpdateDependentGraph_ Should the internal cache of reverse dependents be updated.
274  /// @details See @ref Mezzanine::Threading::FrameScheduler::DependentGraph "DependentGraph"
275  /// for the appropriate times to use this.
276  virtual void SortWorkUnitsMain(bool UpdateDependentGraph_ = true);
277 
278  /// @brief Sort the WorkUnits that must run on the main thread to allow them to be used more efficiently in the next frame executed.
279  /// @param UpdateDependentGraph_ Should the internal cache of reverse dependents be updated.
280  /// @details See @ref Mezzanine::Threading::FrameScheduler::DependentGraph "DependentGraph"
281  /// for the appropriate times to use this.
282  virtual void SortWorkUnitsAffinity(bool UpdateDependentGraph_ = true);
283 
284  /// @brief Sort all the WorkUnits that must run on the main thread to allow them to be used more efficiently in the next frame executed.
285  /// @param UpdateDependentGraph_ Should the internal cache of reverse dependents be updated.
286  /// @details See @ref Mezzanine::Threading::FrameScheduler::DependentGraph "DependentGraph"
287  /// for the appropriate times to use this.
288  virtual void SortWorkUnitsAll(bool UpdateDependentGraph_ = true);
289 
290  // @brief Remove a WorkUnit regardless of type, and caller regains ownership of it.
291  // @param LessWork A pointer to a WorkUnit that should no longer be scheduled.
292  // @details This is relative slow compared to adding or finding a working unit, this works in linear time relative to the number of WorkUnits in the scheduler.
293  //virtual void RemoveWorkUnit(iWorkUnit* LessWork);
294 
295  /// @brief Remove a WorkUnit from the main pool of WorkUnits (and not from the groups of Affinity or MonpolyWorkUnits).
296  /// @param LessWork A pointer to the workunit the calling coding will reclaim ownership of and will no longer be scheduled, and have its dependencies removed.
297  virtual void RemoveWorkUnitMain(iWorkUnit* LessWork);
298 
299  /// @brief Remove a WorkUnit from the Affinity pool of WorkUnits (and not from the Main group or MonpolyWorkUnits).
300  /// @param LessWork A pointer to the workunit the calling coding will reclaim ownership of and will no longer be scheduled, and have its dependencies removed.
301  virtual void RemoveWorkUnitAffinity(iWorkUnit* LessWork);
302 
303  /// @brief Remove a WorkUnit from the Monopoly pool of WorkUnits (and not from the Main or Affinity group).
304  /// @param LessWork A pointer to the MonopolyWorkUnit the calling coding will reclaim ownership of and will no longer be scheduled, and have its dependencies removed.
305  virtual void RemoveWorkUnitMonopoly(MonopolyWorkUnit* LessWork);
306 
307  ////////////////////////////////////////////////////////////////////////////////
308  // Algorithm essentials
309 
310  /// @brief How many other WorkUnit instances must wait on this one.
311  /// @param Work The WorkUnit to get the updated count of.
312  /// @param UsedCachedDepedentGraph If the cache is already up to date leaving this false, and not updating it can save significant time.
313  /// @return A Whole Number representing the amount of WorkUnit instances that cannot start until this finishes.
314  virtual Whole GetDependentCountOf(iWorkUnit *Work, bool UsedCachedDepedentGraph=false);
315 
316  /// @brief Gets the next available workunit for execution.
317  /// @details This finds the next available WorkUnit which has not started execution, has no dependencies that have
318  /// not complete, has the most WorkUnits that depend on it (has the highest runtime in the case of a tie).
319  /// @return A pointer to the WorkUnit that could be executed or a null pointer if that could not be acquired. This does not give ownership of that WorkUnit.
320  virtual iWorkUnit* GetNextWorkUnit();
321 
322  /// @brief Just like @ref GetNextWorkUnit except that it also searches through and prioritizes work units with affinity too.
323  /// @return A pointer to the WorkUnit that could be executed *in the main thread* or a null pointer if that could not be acquired. This does not give ownership of that WorkUnit.
324  virtual iWorkUnit* GetNextWorkUnitAffinity();
325 
326  /// @brief Is the work of the frame done?
327  /// @return This returns true if all the WorkUnit instances are complete, and false otherwise.
328  virtual bool AreAllWorkUnitsComplete();
329 
330  /// @brief Create a reverse depedent graph that can be used for sorting Mezzanine::Threading::iWorkUnit "iWorkUnit"s to optimize execution each frame.
331  /// @details This can be called automatically from any of several places that make sense by passing a boolean true value.
332  /// These place include create a @ref WorkUnitKey or Sorting the work units in a framescheduler.
333  virtual void UpdateDependentGraph();
334 
335  ////////////////////////////////////////////////////////////////////////////////
336  // Algorithm Configuration and Introspection
337 
338  /// @brief Get the current number of frames that have elapsed
339  /// @return A Whole containing the frame count.
340  virtual Whole GetFrameCount() const;
341 
342  /// @brief Get the desired length of a frame.
343  /// @return The desired frame length as a Whole in Microseconds
344  virtual Whole GetFrameLength() const;
345 
346  /// @brief Set the desired Frate rate.
347  /// @param FrameRate in frames per second
348  /// @details Defaults to 60, to maximize smoothmess of execution (no human can see that fast),
349  /// while not killing battery life. This is a maximum framerate, if WorkUnits take too long
350  /// to execute this will not make them finish faster. This controls a delay to prevent the
351  /// machine's resources from being completely tapped. @n @n Set this to 0 to never pause
352  /// and run as fast as possible.
353  /// @return A Whole containing the Target frame rate.
354  virtual void SetFrameRate(const Whole& FrameRate);
355 
356  /// @brief Set the Desired length of a frame in microseconds.
357  /// @param FrameLength The desired minimum length of the frame. Use 0 for no pause.
358  virtual void SetFrameLength(const Whole& FrameLength);
359 
360  /// @brief Get the amount of threads that will be used to execute WorkUnits a the start of the next frame.
361  /// @return A Whole with the current desired thread count.
362  virtual Whole GetThreadCount();
363 
364  /// @brief Set the amount of threads to use.
365  /// @param NewThreadCount The amount of threads to use starting at the begining of the next frame.
366  /// @note Currently the thread count cannot be reduced if Mezz_MinimizeThreadsEachFrame is selected in cmake configuration.
367  virtual void SetThreadCount(const Whole& NewThreadCount);
368 
369  /// @brief When did this frame start?
370  /// @return A MaxInt with the timestamp corresponding to when this frame started.
371  virtual MaxInt GetCurrentFrameStart() const;
372 
373  /// @brief Get The complete record of the past durations of the Pauses Each frame.
374  /// @return A Rolling average of the default type.
375  DefaultRollingAverage<Whole>::Type& GetPauseTimeRollingAverage();
376 
377  /// @brief How long was the pause, if any, last frame?
378  /// @return A whole containing the duration of the last pause.
379  Whole GetLastPauseTime() const;
380 
381  /// @brief Get The complete record of the durations of the last few frames.
382  /// @return A Rolling average of the default type.
383  DefaultRollingAverage<Whole>::Type& GetFrameTimeRollingAverage();
384 
385  /// @brief How long was the previous frame?
386  /// @return A Whole containing the duration of the last frame.
387  Whole GetLastFrameTime() const;
388 
389 
390  ////////////////////////////////////////////////////////////////////////////////
391  // Executing a Frame
392 
393  /// @brief Do one frame worth of work.
394  /// @details This just calls the following functions in the order presented:
395  /// @code
396  /// void FrameScheduler::DoOneFrame()
397  /// {
398  /// RunAllMonopolies();
399  /// CreateThreads();
400  /// RunMainThreadWork();
401  /// JoinAllThreads();
402  /// ResetAllWorkUnits();
403  /// WaitUntilNextFrame();
404  /// }
405  /// @endcode
406  /// This can be replaced calling these functions in this order. You can add any other calls you like between the various stages. This can be done to
407  /// allow maximum integration with existing projects. It can also be used to prevent a giant migration and replace it with a piecemeal upgrade.
408  /// @warning Do not call this on an unsorted set of WorkUnits. Use @ref FrameScheduler::SortWorkUnitsAll() and the @ref DependentGraph to sort WorkUnits after
409  /// they are inserted into the frame scheduler for the first time. This doesn't need to happen each frame, just after any new work units are added or removed
410  /// (except Monopolies) or you want to take the most recent performance number into account.
411  virtual void DoOneFrame();
412 
413  // Image that the next 6 functions should be call in sequence for ideal performance.
414  // Do preparation here.
415 
416  // Setup anything monopolies need, though it would be better if the monopolies could do it themselves in a multithreaded way.
417 
418  /// @brief This is the 1st step (of 6) in a frame.
419  /// @details This iterates over the listing of @ref MonopolyWorkUnit "MonopolyWorkUnit"s and executes each one in the order
420  /// it was added. This should be considered as consuming all available CPU time until it returns. This call blocks until execution
421  /// of monopolies is complete.
422  virtual void RunAllMonopolies();
423 
424  // If the monopolies need some cleanup, it could be done here. Not much should be done here, most work is better as a work unit.
425 
426  /// @brief This is the 2nd step (of 6) in a frame.
427  /// @details This starts all the threads on their work. Until @ref JoinAllThreads() is called some thread may still be working.
428  /// This thread starts the man @ref algorithm_sec "scheduling algorithm" working on every thread except the calling thread. This
429  /// call does not block and tends to return very quickly.
430  /// @n @n
431  /// This checks the amount of threads as set by @ref SetFrameLength "SetFrameLength". It creates any
432  /// @ref ThreadSpecificStorage instances required and creates threads if they are required. SwapAllBufferedResources() Is called on
433  /// each ThreadSpecificStorage before being passed into the thread. If extra double buffered resources are required per thread
434  /// and they need to be swapped each frame, SwapAllBufferedResources() should be inherited or adjusted to account for these new
435  /// resources.
436  /// @n @n
437  /// If the build option
438  /// @ref MEZZ_USEBARRIERSEACHFRAME Mezz_MinimizeThreadsEachFrame was enabled then this will reuse threads from previous frames,
439  /// otherwise this will re-use thread specific resources and create a new set of threads. Re-use of threads is synchronized
440  /// with the @ref Barrier StartFrameSync member variable. It is unclear, and likely platform specific, which option has
441  /// better performance characteristics.
442  /// @warning While this is running any changes to the @ref FrameScheduler must be made with an atomic operation like the
443  /// @ref AtomicCompareAndSwap32 "AtomicCompareAndSwap32" or @ref AtomicAdd "AtomicAdd". Any other threads
444  /// workunit may be accessed as any normal shared data, but Thread specific Resources should not be accessed while this runs.
445  /// @warning This uses a Spinlock to prevent accesss to ThreadSpecificResources that the LogAggregator needs. This is unlocked
446  /// in RunMainThreadWork.
447  virtual void CreateThreads();
448 
449  // If it must be run on the main thread and must be run first each frame it could go here, but I think you should
450  // just make a work unit out of it and make other work units depend on it. Right now all the other thread are working.
451 
452  /// @brief This is the 3rd step (of 6) in a frame.
453  /// @details This runs the main portion of the @ref algorithm_sec "scheduling algorithm" on the main thread. This call
454  /// blocks until the execution of all workunits with main thread affinity are complete and all other work units have at
455  /// least started. This could return and other threads could still be working.
456  /// @n @n
457  /// Before executing any work this checks a flag to determine if it should log the current work unit dependencies. After
458  /// this check or logging has occurred this then this releases the spinlock on the log, so there is a chance of brief
459  /// contention between a LogAggregator and this if depedencies have just changed or change frequently.
460  /// @warning This uses a Spinlock to prevent accesss to ThreadSpecificResources that the LogAggregator needs. This should
461  /// be called immediately after CreateThreads to minimize any possible contention.
462  virtual void RunMainThreadWork();
463 
464  // Alled the work units have at least been started, and quite possible most have finished. It is possible a few threads
465  // have finished. But if you must do something after all the workunits have started and most finished, while the threads
466  // remain in memory, then this is the place to do it.
467 
468  /// @brief This is the 4th step (of 6) in a frame.
469  /// @details Used when completing the work of a frame, to cleaning end the execution of the threads. This function will
470  /// only return when all the work started by @ref CreateThreads() and @ref RunMainThreadWork() have completed. This call
471  /// blocks until all threads executing. If a thread takes too long then this simply waits for it to finish. No attempt is
472  /// made to timeout or interupt a work unit before it finishes.
473  void JoinAllThreads();
474 
475  // All the threads have been cleaned up or paused. All the work units have finished. This is a reasonable place for heuristics
476  // and maybe other kinds of work. but maybe that stuff could go in a monopoly instead. This might be a good place to check
477  // statuses of work units, but all the interesting statuses kill the program.
478 
479  /// @brief This is the 5th step (of 6) in a frame.
480  /// @details Take any steps required to prepare all owned WorkUnits for execution next frame. This usually includes reseting
481  /// all the work units running state to @ref NotStarted "NotStarted". This can cause work units to be executed multiple times
482  /// if a thread is still executing.
483  virtual void ResetAllWorkUnits();
484 
485  // All the work units are ready for the next frame, but no real waiting has occurred yet.
486 
487  /// @brief This is the final step (of 6) in a frame.
488  /// @details Wait until this frame has consumed its fair share of a second. This uses the value passed in
489  /// @ref SetFrameRate "SetFrameRate" to determine what portion of a second each frame should
490  /// use. If a frame took too long to execute this calculates that and returns.
491  /// @n @n
492  /// Wait 1/TargetFrame seconds, minus time already run. This also starts the timer for the next frame so
493  /// any other logic that needs to run after the frame does not interfere with frame timing. Because
494  /// This is designed to wait fractions of a second any amount of waiting above 1 second fails automaticall.
495  void WaitUntilNextFrame();
496 
497  // This is the same place as before the monopolies, except for whatever one time setup code you might have.
498 
499  ////////////////////////////////////////////////////////////////////////////////
500  // Basic container features
501 
502  /// @brief Returns the amount of MonopolyWorkUnit ready to be scheduled
503  /// @return A Whole containing this amount
504  Whole GetWorkUnitMonopolyCount() const;
505 
506  /// @brief Returns the amount of iWorkUnit ready to be scheduled in the Affinity pool
507  /// @return A Whole containing this amount
508  Whole GetWorkUnitAffinityCount() const;
509 
510  /// @brief Returns the amount of iWorkUnit ready to be scheduled in the Main pool
511  /// @return A Whole containing this amount
512  Whole GetWorkUnitMainCount() const;
513 
514  ////////////////////////////////////////////////////////////////////////////////
515  // Logging Features
516 
517  /// @brief Get the Resource to go with a thread of a given ID.
518  /// @details This gets the Resource that goes with a given thread
519  /// by performing a linear search through the available threads.
520  /// @n @n
521  /// This is cheap computationly but will likely perform much slower
522  /// than other methods of getting the resource, because the ID is
523  /// unlikely be be cached and it may require a system call to
524  /// retrieve. If used consider wrapping it in \#ifdef MEZZ_DEBUG + \#endif
525  /// to disable for release builds.
526  /// @param ID This uses the current Threads ID by default but can search for any thread.
527  /// @return A pointer to ThreadSpecificResource or a null pointer on error.
528  /// @warning The thread that 'owns' this resource could do just about anything
529  /// with it while the frame is running, so this should only outside a frame
530  /// and carefully or inside a frame and only from the owning thread.
531  Resource* GetThreadResource(ThreadId ID = this_thread::get_id());
532 
533  /// @brief Get the logger safe to use this thread.
534  /// @warning This is written in terms of GetThreadResource and has all the
535  /// same limitations.
536  /// @param ID This uses the current Threads ID by default but can search for any thread.
537  /// @return A null pointer if there is an error or a pointer to the Logger that goes with the passed Thread::Id
538  Logger* GetThreadUsableLogger(ThreadId ID = this_thread::get_id());
539 
540  /// @brief Indicate to the framescheduler if dependencies need to be logged
541  /// @param Changed Defaults to true, and sets a flag that tells the framescheduler if it needs to log dependencies.
542  /// @details If false is passed, this prevents the FrameScheduler from logging until the next change.
543  void DependenciesChanged(bool Changed=true);
544 
545  /// @brief This sends the dependencies to the LogDestination (Skipping any thread specific resources)
546  /// @warning This should not be executed during the frame, unless the FrameScheduler is calling it.
547  void LogDependencies();
548 
549  /// @warning This is not thread safe at all. Any time during the frame using this can send gibberish to the log. Use GetThreadUsableLogger instead.
550  /// @brief Get the endpoint for the logs.
551  /// @return An std:ostream reference which can be streamed to commit log entries.
552  std::ostream& GetLog();
553  };//FrameScheduler
554  } // \Threading
555 }// \Mezanine
556 
557 
558 #endif