ogb912/opensim
1
0
Fork 0
mirror of https://github.com/opensim/opensim.git synced 2026-08-04 16:22:50 +08:00
Code Issues Packages Projects Releases Wiki Activity

Implemented a Watchdog class. Do not manually create Thread objects anymore, use Watchdog.StartThread(). While your thread is running call Watchdog.UpdateThread(). When it is shutting down call Watchdog.RemoveThread(). Most of the threads in OpenSim have been updated

Browse Source
This commit is contained in:
John Hurliman
2009-10-22 12:33:23 -07:00
parent 11013ad295
commit b2ed348aa2
17 changed files with 309 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
OpenSim/Region/Application/OpenSim.cs
View File

@@ -172,6 +172,9 @@ namespace OpenSim
m_scriptTimer.Elapsed += RunAutoTimerScript;
}
// Hook up to the watchdog timer
Watchdog.OnWatchdogTimeout += WatchdogTimeoutHandler;
PrintFileToConsole("startuplogo.txt");
// For now, start at the 'root' level by default
@@ -384,6 +387,14 @@ namespace OpenSim
}
}
private void WatchdogTimeoutHandler(System.Threading.Thread thread, int lastTick)
{
int now = Environment.TickCount & Int32.MaxValue;
m_log.ErrorFormat("[WATCHDOG]: Timeout detected for thread \"{0}\". ThreadState={1}. Last tick was {2}ms ago",
thread.Name, thread.ThreadState, now - lastTick);
}
#region Console Commands
/// <summary>

19
OpenSim/Region/ClientStack/LindenUDP/LLUDPServer.cs
View File

@@ -187,14 +187,9 @@ namespace OpenSim.Region.ClientStack.LindenUDP
base.Start(m_recvBufferSize, m_asyncPacketHandling);
// Start the incoming packet processing thread
Thread incomingThread = new Thread(IncomingPacketHandler);
incomingThread.Name = "Incoming Packets (" + m_scene.RegionInfo.RegionName + ")";
incomingThread.Start();
Thread outgoingThread = new Thread(OutgoingPacketHandler);
outgoingThread.Name = "Outgoing Packets (" + m_scene.RegionInfo.RegionName + ")";
outgoingThread.Start();
// Start the packet processing threads
Watchdog.StartThread(IncomingPacketHandler, "Incoming Packets (" + m_scene.RegionInfo.RegionName + ")", ThreadPriority.Normal, false);
Watchdog.StartThread(OutgoingPacketHandler, "Outgoing Packets (" + m_scene.RegionInfo.RegionName + ")", ThreadPriority.Normal, false);
}
public new void Stop()
@@ -775,11 +770,15 @@ namespace OpenSim.Region.ClientStack.LindenUDP
{
m_log.Error("[LLUDPSERVER]: Error in the incoming packet handler loop: " + ex.Message, ex);
}
Watchdog.UpdateThread();
}
if (packetInbox.Count > 0)
m_log.Warn("[LLUDPSERVER]: IncomingPacketHandler is shutting down, dropping " + packetInbox.Count + " packets");
packetInbox.Clear();
Watchdog.RemoveThread();
}
private void OutgoingPacketHandler()
@@ -842,12 +841,16 @@ namespace OpenSim.Region.ClientStack.LindenUDP
// token bucket could get more tokens
if (!m_packetSent)
Thread.Sleep((int)TickCountResolution);
Watchdog.UpdateThread();
}
catch (Exception ex)
{
m_log.Error("[LLUDPSERVER]: OutgoingPacketHandler loop threw an exception: " + ex.Message, ex);
}
}
Watchdog.RemoveThread();
}
private void ClientOutgoingPacketHandler(IClientAPI client)

15
OpenSim/Region/CoreModules/InterGrid/OpenGridProtocolModule.cs
View File

@@ -1208,10 +1208,7 @@ namespace OpenSim.Region.CoreModules.InterGrid
if (homeScene.TryGetAvatar(avatarId,out avatar))
{
KillAUser ku = new KillAUser(avatar,mod);
Thread ta = new Thread(ku.ShutdownNoLogout);
ta.IsBackground = true;
ta.Name = "ShutdownThread";
ta.Start();
Watchdog.StartThread(ku.ShutdownNoLogout, "OGPShutdown", ThreadPriority.Normal, true);
}
}
@@ -1261,7 +1258,13 @@ namespace OpenSim.Region.CoreModules.InterGrid
avToBeKilled.ControllingClient.SendLogoutPacketWhenClosing = false;
Thread.Sleep(30000);
int sleepMS = 30000;
while (sleepMS > 0)
{
Watchdog.UpdateThread();
Thread.Sleep(1000);
sleepMS -= 1000;
}
// test for child agent because they might have come back
if (avToBeKilled.IsChildAgent)
@@ -1270,6 +1273,8 @@ namespace OpenSim.Region.CoreModules.InterGrid
avToBeKilled.ControllingClient.Close();
}
}
Watchdog.RemoveThread();
}
}

8
OpenSim/Region/CoreModules/World/Archiver/AssetsRequest.cs
View File

@@ -128,7 +128,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
if (m_repliesRequired == 0)
{
m_requestState = RequestState.Completed;
PerformAssetsRequestCallback();
PerformAssetsRequestCallback(null);
return;
}
@@ -246,9 +246,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
// We want to stop using the asset cache thread asap
// as we now need to do the work of producing the rest of the archive
Thread newThread = new Thread(PerformAssetsRequestCallback);
newThread.Name = "OpenSimulator archiving thread post assets receipt";
newThread.Start();
Util.FireAndForget(PerformAssetsRequestCallback);
}
else
{
@@ -265,7 +263,7 @@ namespace OpenSim.Region.CoreModules.World.Archiver
/// <summary>
/// Perform the callback on the original requester of the assets
/// </summary>
protected void PerformAssetsRequestCallback()
protected void PerformAssetsRequestCallback(object o)
{
try
{

13
OpenSim/Region/CoreModules/World/WorldMap/WorldMapModule.cs
View File

@@ -74,7 +74,6 @@ namespace OpenSim.Region.CoreModules.World.WorldMap
private Dictionary<ulong, int> m_blacklistedregions = new Dictionary<ulong, int>();
private Dictionary<ulong, string> m_cachedRegionMapItemsAddress = new Dictionary<ulong, string>();
private List<UUID> m_rootAgents = new List<UUID>();
private Thread mapItemReqThread;
private volatile bool threadrunning = false;
//private int CacheRegionsDistance = 256;
@@ -338,13 +337,10 @@ namespace OpenSim.Region.CoreModules.World.WorldMap
{
if (threadrunning) return;
threadrunning = true;
m_log.Debug("[WORLD MAP]: Starting remote MapItem request thread");
mapItemReqThread = new Thread(new ThreadStart(process));
mapItemReqThread.IsBackground = true;
mapItemReqThread.Name = "MapItemRequestThread";
mapItemReqThread.Priority = ThreadPriority.BelowNormal;
mapItemReqThread.SetApartmentState(ApartmentState.MTA);
mapItemReqThread.Start();
Watchdog.StartThread(process, "MapItemRequestThread", ThreadPriority.BelowNormal, true);
}
/// <summary>
@@ -461,6 +457,8 @@ namespace OpenSim.Region.CoreModules.World.WorldMap
OSDMap response = RequestMapItemsAsync("", st.agentID, st.flags, st.EstateID, st.godlike, st.itemtype, st.regionhandle);
RequestMapItemsCompleted(response);
}
Watchdog.UpdateThread();
}
}
catch (Exception e)
@@ -469,6 +467,7 @@ namespace OpenSim.Region.CoreModules.World.WorldMap
}
threadrunning = false;
Watchdog.RemoveThread();
}
/// <summary>

20
OpenSim/Region/Framework/Scenes/Scene.cs
View File

@@ -81,8 +81,6 @@ namespace OpenSim.Region.Framework.Scenes
protected Timer m_restartWaitTimer = new Timer();
protected Thread m_updateEntitiesThread;
public SimStatsReporter StatsReporter;
protected List<RegionInfo> m_regionRestartNotifyList = new List<RegionInfo>();
@@ -945,11 +943,8 @@ namespace OpenSim.Region.Framework.Scenes
HeartbeatThread = null;
}
m_lastUpdate = Environment.TickCount;
HeartbeatThread = new Thread(new ParameterizedThreadStart(Heartbeat));
HeartbeatThread.SetApartmentState(ApartmentState.MTA);
HeartbeatThread.Name = string.Format("Heartbeat for region {0}", RegionInfo.RegionName);
HeartbeatThread.Priority = ThreadPriority.AboveNormal;
HeartbeatThread.Start();
HeartbeatThread = Watchdog.StartThread(Heartbeat, "Heartbeat for region " + RegionInfo.RegionName, ThreadPriority.Normal, false);
}
/// <summary>
@@ -976,12 +971,13 @@ namespace OpenSim.Region.Framework.Scenes
/// <summary>
/// Performs per-frame updates regularly
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void Heartbeat(object sender)
private void Heartbeat()
{
if (!Monitor.TryEnter(m_heartbeatLock))
{
Watchdog.RemoveThread();
return;
}
try
{
@@ -998,6 +994,8 @@ namespace OpenSim.Region.Framework.Scenes
Monitor.Pulse(m_heartbeatLock);
Monitor.Exit(m_heartbeatLock);
}
Watchdog.RemoveThread();
}
/// <summary>
@@ -1146,6 +1144,8 @@ namespace OpenSim.Region.Framework.Scenes
if ((maintc < (m_timespan * 1000)) && maintc > 0)
Thread.Sleep(maintc);
Watchdog.UpdateThread();
}
}

10
OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCClientView.cs
View File

@@ -67,9 +67,8 @@ namespace OpenSim.Region.OptionalModules.Agent.InternetRelayClientView.Server
{
m_client = client;
m_scene = scene;
Thread loopThread = new Thread(InternalLoop);
loopThread.Start();
Watchdog.StartThread(InternalLoop, "IRCClientView", ThreadPriority.Normal, false);
}
private void SendServerCommand(string command)
@@ -102,7 +101,7 @@ namespace OpenSim.Region.OptionalModules.Agent.InternetRelayClientView.Server
{
try
{
string strbuf = "";
string strbuf = String.Empty;
while (m_connected && m_client.Connected)
{
@@ -140,6 +139,7 @@ namespace OpenSim.Region.OptionalModules.Agent.InternetRelayClientView.Server
}
Thread.Sleep(0);
Watchdog.UpdateThread();
}
}
catch (IOException)
@@ -156,6 +156,8 @@ namespace OpenSim.Region.OptionalModules.Agent.InternetRelayClientView.Server
m_log.Warn("[IRCd] Disconnected client.");
}
Watchdog.RemoveThread();
}
private void ProcessInMessage(string message, string command)

7
OpenSim/Region/OptionalModules/Agent/InternetRelayClientView/Server/IRCServer.cs
View File

@@ -33,6 +33,7 @@ using System.Reflection;
using System.Text;
using System.Threading;
using log4net;
using OpenSim.Framework;
using OpenSim.Region.Framework.Scenes;
namespace OpenSim.Region.OptionalModules.Agent.InternetRelayClientView.Server
@@ -56,8 +57,7 @@ namespace OpenSim.Region.OptionalModules.Agent.InternetRelayClientView.Server
m_listener.Start(50);
Thread thread = new Thread(ListenLoop);
thread.Start();
Watchdog.StartThread(ListenLoop, "IRCServer", ThreadPriority.Normal, false);
m_baseScene = baseScene;
}
@@ -72,7 +72,10 @@ namespace OpenSim.Region.OptionalModules.Agent.InternetRelayClientView.Server
while (m_running)
{
AcceptClient(m_listener.AcceptTcpClient());
Watchdog.UpdateThread();
}
Watchdog.RemoveThread();
}
private void AcceptClient(TcpClient client)

10
OpenSim/Region/OptionalModules/ContentManagementSystem/CMController.cs
View File

@@ -86,7 +86,6 @@ namespace OpenSim.Region.OptionalModules.ContentManagement
/// </value>
Hashtable m_sceneList = Hashtable.Synchronized(new Hashtable());
State m_state = State.NONE;
Thread m_thread = null;
CMView m_view = null;
#endregion Fields
@@ -148,10 +147,7 @@ namespace OpenSim.Region.OptionalModules.ContentManagement
lock (this)
{
m_estateModule = scene.RequestModuleInterface<IEstateModule>();
m_thread = new Thread(MainLoop);
m_thread.Name = "Content Management";
m_thread.IsBackground = true;
m_thread.Start();
Watchdog.StartThread(MainLoop, "Content Management", ThreadPriority.Normal, true);
m_state = State.NONE;
}
}
@@ -200,6 +196,8 @@ namespace OpenSim.Region.OptionalModules.ContentManagement
m_log.Debug("[CONTENT MANAGEMENT] MAIN LOOP -- uuuuuuuuuh, what?");
break;
}
Watchdog.UpdateThread();
}
}
catch (Exception e)
@@ -209,6 +207,8 @@ namespace OpenSim.Region.OptionalModules.ContentManagement
"[CONTENT MANAGEMENT]: Content management thread terminating with exception. PLEASE REBOOT YOUR SIM - CONTENT MANAGEMENT WILL NOT BE AVAILABLE UNTIL YOU DO. Exception is {0}",
e);
}
Watchdog.RemoveThread();
}
/// <summary>

12
OpenSim/Region/ScriptEngine/DotNetEngine/EventQueueThreadClass.cs
View File

@@ -132,12 +132,7 @@ namespace OpenSim.Region.ScriptEngine.DotNetEngine
/// </summary>
private void Start()
{
EventQueueThread = new Thread(EventQueueThreadLoop);
EventQueueThread.IsBackground = true;
EventQueueThread.Priority = MyThreadPriority;
EventQueueThread.Name = "EventQueueManagerThread_" + ThreadCount;
EventQueueThread.Start();
EventQueueThread = Watchdog.StartThread(EventQueueThreadLoop, "EventQueueManagerThread_" + ThreadCount, MyThreadPriority, true);
// Look at this... Don't you wish everyone did that solid
// coding everywhere? :P
@@ -184,6 +179,7 @@ namespace OpenSim.Region.ScriptEngine.DotNetEngine
while (true)
{
DoProcessQueue();
Watchdog.UpdateThread();
}
}
catch (ThreadAbortException)
@@ -214,6 +210,8 @@ namespace OpenSim.Region.ScriptEngine.DotNetEngine
m_log.ErrorFormat("[{0}]: Exception {1} thrown", ScriptEngineName, e.GetType().ToString());
throw e;
}
Watchdog.UpdateThread();
}
}
catch (ThreadAbortException)
@@ -226,6 +224,8 @@ namespace OpenSim.Region.ScriptEngine.DotNetEngine
"[{0}]: Event queue thread terminating with exception. PLEASE REBOOT YOUR SIM - SCRIPT EVENTS WILL NOT WORK UNTIL YOU DO. Exception is {1}",
ScriptEngineName, e);
}
Watchdog.RemoveThread();
}
public void DoProcessQueue()

89
OpenSim/Region/ScriptEngine/DotNetEngine/MaintenanceThread.cs
View File

@@ -93,10 +93,7 @@ namespace OpenSim.Region.ScriptEngine.DotNetEngine
{
if (MaintenanceThreadThread == null)
{
MaintenanceThreadThread = new Thread(MaintenanceLoop);
MaintenanceThreadThread.Name = "ScriptMaintenanceThread";
MaintenanceThreadThread.IsBackground = true;
MaintenanceThreadThread.Start();
MaintenanceThreadThread = Watchdog.StartThread(MaintenanceLoop, "ScriptMaintenanceThread", ThreadPriority.Normal, true);
}
}
@@ -164,56 +161,54 @@ namespace OpenSim.Region.ScriptEngine.DotNetEngine
MaintenanceLoopTicks_ScriptLoadUnload_Count++;
MaintenanceLoopTicks_Other_Count++;
//lock (ScriptEngine.ScriptEngines)
//{
foreach (ScriptEngine m_ScriptEngine in new ArrayList(ScriptEngine.ScriptEngines))
foreach (ScriptEngine m_ScriptEngine in new ArrayList(ScriptEngine.ScriptEngines))
{
// lastScriptEngine = m_ScriptEngine;
// Re-reading config every x seconds
if (MaintenanceLoopTicks_Other_Count >= MaintenanceLoopTicks_Other)
{
// lastScriptEngine = m_ScriptEngine;
// Re-reading config every x seconds
if (MaintenanceLoopTicks_Other_Count >= MaintenanceLoopTicks_Other)
MaintenanceLoopTicks_Other_ResetCount = true;
if (m_ScriptEngine.RefreshConfigFilens > 0)
{
MaintenanceLoopTicks_Other_ResetCount = true;
if (m_ScriptEngine.RefreshConfigFilens > 0)
// Check if its time to re-read config
if (DateTime.Now.Ticks - Last_ReReadConfigFilens >
m_ScriptEngine.RefreshConfigFilens)
{
// Check if its time to re-read config
if (DateTime.Now.Ticks - Last_ReReadConfigFilens >
m_ScriptEngine.RefreshConfigFilens)
//m_log.Debug("Time passed: " + (DateTime.Now.Ticks - Last_ReReadConfigFilens) + ">" + m_ScriptEngine.RefreshConfigFilens);
// Its time to re-read config file
m_ScriptEngine.ReadConfig();
Last_ReReadConfigFilens = DateTime.Now.Ticks; // Reset time
}
// Adjust number of running script threads if not correct
if (m_ScriptEngine.m_EventQueueManager != null)
m_ScriptEngine.m_EventQueueManager.AdjustNumberOfScriptThreads();
// Check if any script has exceeded its max execution time
if (EventQueueManager.EnforceMaxExecutionTime)
{
// We are enforcing execution time
if (DateTime.Now.Ticks - Last_maxFunctionExecutionTimens >
EventQueueManager.maxFunctionExecutionTimens)
{
//m_log.Debug("Time passed: " + (DateTime.Now.Ticks - Last_ReReadConfigFilens) + ">" + m_ScriptEngine.RefreshConfigFilens);
// Its time to re-read config file
m_ScriptEngine.ReadConfig();
Last_ReReadConfigFilens = DateTime.Now.Ticks; // Reset time
}
// Adjust number of running script threads if not correct
if (m_ScriptEngine.m_EventQueueManager != null)
m_ScriptEngine.m_EventQueueManager.AdjustNumberOfScriptThreads();
// Check if any script has exceeded its max execution time
if (EventQueueManager.EnforceMaxExecutionTime)
{
// We are enforcing execution time
if (DateTime.Now.Ticks - Last_maxFunctionExecutionTimens >
EventQueueManager.maxFunctionExecutionTimens)
{
// Its time to check again
m_ScriptEngine.m_EventQueueManager.CheckScriptMaxExecTime(); // Do check
Last_maxFunctionExecutionTimens = DateTime.Now.Ticks; // Reset time
}
// Its time to check again
m_ScriptEngine.m_EventQueueManager.CheckScriptMaxExecTime(); // Do check
Last_maxFunctionExecutionTimens = DateTime.Now.Ticks; // Reset time
}
}
}
if (MaintenanceLoopTicks_ScriptLoadUnload_Count >= MaintenanceLoopTicks_ScriptLoadUnload)
{
MaintenanceLoopTicks_ScriptLoadUnload_ResetCount = true;
// LOAD / UNLOAD SCRIPTS
if (m_ScriptEngine.m_ScriptManager != null)
m_ScriptEngine.m_ScriptManager.DoScriptLoadUnload();
}
}
//}
if (MaintenanceLoopTicks_ScriptLoadUnload_Count >= MaintenanceLoopTicks_ScriptLoadUnload)
{
MaintenanceLoopTicks_ScriptLoadUnload_ResetCount = true;
// LOAD / UNLOAD SCRIPTS
if (m_ScriptEngine.m_ScriptManager != null)
m_ScriptEngine.m_ScriptManager.DoScriptLoadUnload();
}
}
Watchdog.UpdateThread();
}
}
catch(ThreadAbortException)
@@ -225,6 +220,8 @@ namespace OpenSim.Region.ScriptEngine.DotNetEngine
m_log.ErrorFormat("Exception in MaintenanceLoopThread. Thread will recover after 5 sec throttle. Exception: {0}", ex.ToString());
}
}
Watchdog.RemoveThread();
}
#endregion

8
OpenSim/Region/ScriptEngine/Shared/Api/Implementation/AsyncCommandManager.cs
View File

@@ -137,11 +137,7 @@ namespace OpenSim.Region.ScriptEngine.Shared.Api
if (cmdHandlerThread == null)
{
// Start the thread that will be doing the work
cmdHandlerThread = new Thread(CmdHandlerThreadLoop);
cmdHandlerThread.Name = "AsyncLSLCmdHandlerThread";
cmdHandlerThread.Priority = ThreadPriority.BelowNormal;
cmdHandlerThread.IsBackground = true;
cmdHandlerThread.Start();
cmdHandlerThread = Watchdog.StartThread(CmdHandlerThreadLoop, "AsyncLSLCmdHandlerThread", ThreadPriority.Normal, true);
}
}
@@ -185,6 +181,8 @@ namespace OpenSim.Region.ScriptEngine.Shared.Api
Thread.Sleep(cmdHandlerThreadCycleSleepms);
DoOneCmdHandlerPass();
Watchdog.UpdateThread();
}
}
catch