Updated deadlock detector.
This commit is contained in:
parent
ee84ae87fa
commit
6b5c3c6d21
@ -134,6 +134,21 @@ InstantThreadPoolCount = 120
|
||||
UrgentPacketThreadCoreSize = 40
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dead Lock Detector (separate thread for detecting deadlocks)
|
||||
# ---------------------------------------------------------------------------
|
||||
# For improved crash logs and automatic restart in deadlock case if enabled.
|
||||
# Check interval is in seconds.
|
||||
# Default: True
|
||||
DeadLockDetector = True
|
||||
|
||||
# Default: 20
|
||||
DeadLockCheckInterval = 20
|
||||
|
||||
# Default: False
|
||||
RestartOnDeadlock = False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Misc Player Settings
|
||||
# ---------------------------------------------------------------------------
|
||||
|
@ -1,20 +0,0 @@
|
||||
# ---------------------------------------------------------------------------
|
||||
# Daemons settings
|
||||
# ---------------------------------------------------------------------------
|
||||
# Preamble:
|
||||
# Standard set up is possible only two parameters of demons:
|
||||
# - Pereodichnost run in milliseconds
|
||||
# - Delay in milliseconds before starting
|
||||
# Zero or the number of negative values of this trip of a parameter
|
||||
# For example:
|
||||
# DeadLockCheck = -1
|
||||
# Now the demon-stream after completing his tasks will disappear forever.
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dead Lock Detector (separate thread for detecting deadlocks)
|
||||
# ---------------------------------------------------------------------------
|
||||
# For improved crash logs and automatic restart in deadlock case if enabled.
|
||||
# Check interval is in seconds.
|
||||
# Default: 0 (disabled)
|
||||
DeadLockCheck = 0
|
||||
DeadLockDelay = 0
|
@ -80,7 +80,6 @@ public class Config
|
||||
private static final String SEVENSIGNS_CONFIG_FILE = "./config/main/SevenSigns.ini";
|
||||
public static final String SIEGE_CONFIG_FILE = "./config/main/Siege.ini";
|
||||
// protected
|
||||
private static final String DAEMONS_CONFIG_FILE = "./config/protected/Daemons.ini";
|
||||
private static final String PROTECT_FLOOD_CONFIG_FILE = "./config/protected/Flood.ini";
|
||||
private static final String PROTECT_OTHER_CONFIG_FILE = "./config/protected/Other.ini";
|
||||
public static final String TELNET_CONFIG_FILE = "./config/protected/Telnet.ini";
|
||||
@ -1102,9 +1101,6 @@ public class Config
|
||||
public static boolean ALLOW_REMOTE_CLASS_MASTERS;
|
||||
public static boolean ENABLE_EXP_GAIN_COMMANDS;
|
||||
|
||||
public static long DEADLOCKCHECK_INTIAL_TIME;
|
||||
public static long DEADLOCKCHECK_DELAY_TIME;
|
||||
|
||||
public static List<String> QUESTION_LIST = new ArrayList<>();
|
||||
|
||||
public static int SERVER_ID;
|
||||
@ -1130,6 +1126,9 @@ public class Config
|
||||
public static int SCHEDULED_THREAD_POOL_COUNT;
|
||||
public static int INSTANT_THREAD_POOL_COUNT;
|
||||
public static int IO_PACKET_THREAD_CORE_SIZE;
|
||||
public static boolean DEADLOCK_DETECTOR;
|
||||
public static int DEADLOCK_CHECK_INTERVAL;
|
||||
public static boolean RESTART_ON_DEADLOCK;
|
||||
public static String CNAME_TEMPLATE;
|
||||
public static String PET_NAME_TEMPLATE;
|
||||
public static String CLAN_NAME_TEMPLATE;
|
||||
@ -1222,6 +1221,9 @@ public class Config
|
||||
SCHEDULED_THREAD_POOL_COUNT = serverConfig.getInt("ScheduledThreadPoolCount", 40);
|
||||
INSTANT_THREAD_POOL_COUNT = serverConfig.getInt("InstantThreadPoolCount", 20);
|
||||
IO_PACKET_THREAD_CORE_SIZE = serverConfig.getInt("UrgentPacketThreadCoreSize", 20);
|
||||
DEADLOCK_DETECTOR = serverConfig.getBoolean("DeadLockDetector", true);
|
||||
DEADLOCK_CHECK_INTERVAL = serverConfig.getInt("DeadLockCheckInterval", 20);
|
||||
RESTART_ON_DEADLOCK = serverConfig.getBoolean("RestartOnDeadlock", false);
|
||||
CNAME_TEMPLATE = serverConfig.getString("CnameTemplate", ".*");
|
||||
PET_NAME_TEMPLATE = serverConfig.getString("PetNameTemplate", ".*");
|
||||
CLAN_NAME_TEMPLATE = serverConfig.getString("ClanNameTemplate", ".*");
|
||||
@ -2854,13 +2856,6 @@ public class Config
|
||||
ENABLE_EXP_GAIN_COMMANDS = characterConfig.getBoolean("EnableExpGainCommands", false);
|
||||
}
|
||||
|
||||
public static void loadDaemonsConf()
|
||||
{
|
||||
final PropertiesParser deamonsConfig = new PropertiesParser(DAEMONS_CONFIG_FILE);
|
||||
DEADLOCKCHECK_INTIAL_TIME = deamonsConfig.getLong("DeadLockCheck", 0);
|
||||
DEADLOCKCHECK_DELAY_TIME = deamonsConfig.getLong("DeadLockDelay", 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads all Filter Words
|
||||
*/
|
||||
@ -3158,9 +3153,6 @@ public class Config
|
||||
loadPCBPointConfig();
|
||||
loadOfflineConfig();
|
||||
|
||||
// Other
|
||||
loadDaemonsConf();
|
||||
|
||||
if (USE_SAY_FILTER)
|
||||
{
|
||||
loadFilter();
|
||||
|
@ -0,0 +1,120 @@
|
||||
/*
|
||||
* This file is part of the L2J Mobius project.
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package org.l2jmobius.commons.util;
|
||||
|
||||
import java.lang.management.LockInfo;
|
||||
import java.lang.management.ManagementFactory;
|
||||
import java.lang.management.MonitorInfo;
|
||||
import java.lang.management.ThreadInfo;
|
||||
import java.lang.management.ThreadMXBean;
|
||||
import java.time.Duration;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
|
||||
import org.l2jmobius.Config;
|
||||
|
||||
/**
|
||||
* Thread to check for deadlocked threads.
|
||||
* @author -Nemesiss- L2M
|
||||
*/
|
||||
public class DeadLockDetector extends Thread
|
||||
{
|
||||
private static final Logger LOGGER = Logger.getLogger(DeadLockDetector.class.getName());
|
||||
|
||||
private final Duration _checkInterval;
|
||||
private final Runnable _deadLockCallback;
|
||||
private final ThreadMXBean tmx;
|
||||
|
||||
public DeadLockDetector(Duration checkInterval, Runnable deadLockCallback)
|
||||
{
|
||||
super("DeadLockDetector");
|
||||
_checkInterval = checkInterval;
|
||||
_deadLockCallback = deadLockCallback;
|
||||
tmx = ManagementFactory.getThreadMXBean();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run()
|
||||
{
|
||||
boolean deadlock = false;
|
||||
while (!deadlock)
|
||||
{
|
||||
try
|
||||
{
|
||||
final long[] ids = tmx.findDeadlockedThreads();
|
||||
if (ids != null)
|
||||
{
|
||||
deadlock = true;
|
||||
final ThreadInfo[] tis = tmx.getThreadInfo(ids, true, true);
|
||||
final StringBuilder info = new StringBuilder();
|
||||
info.append("DeadLock Found!");
|
||||
info.append(Config.EOL);
|
||||
for (ThreadInfo ti : tis)
|
||||
{
|
||||
info.append(ti.toString());
|
||||
}
|
||||
|
||||
for (ThreadInfo ti : tis)
|
||||
{
|
||||
final LockInfo[] locks = ti.getLockedSynchronizers();
|
||||
final MonitorInfo[] monitors = ti.getLockedMonitors();
|
||||
if ((locks.length == 0) && (monitors.length == 0))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
ThreadInfo dl = ti;
|
||||
info.append("Java-level deadlock:");
|
||||
info.append(Config.EOL);
|
||||
info.append('\t');
|
||||
info.append(dl.getThreadName());
|
||||
info.append(" is waiting to lock ");
|
||||
info.append(dl.getLockInfo().toString());
|
||||
info.append(" which is held by ");
|
||||
info.append(dl.getLockOwnerName());
|
||||
info.append(Config.EOL);
|
||||
while ((dl = tmx.getThreadInfo(new long[]
|
||||
{
|
||||
dl.getLockOwnerId()
|
||||
}, true, true)[0]).getThreadId() != ti.getThreadId())
|
||||
{
|
||||
info.append('\t');
|
||||
info.append(dl.getThreadName());
|
||||
info.append(" is waiting to lock ");
|
||||
info.append(dl.getLockInfo().toString());
|
||||
info.append(" which is held by ");
|
||||
info.append(dl.getLockOwnerName());
|
||||
info.append(Config.EOL);
|
||||
}
|
||||
}
|
||||
|
||||
LOGGER.warning(info.toString());
|
||||
|
||||
if (_deadLockCallback != null)
|
||||
{
|
||||
_deadLockCallback.run();
|
||||
}
|
||||
}
|
||||
Thread.sleep(_checkInterval.toMillis());
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
LOGGER.log(Level.WARNING, "DeadLockDetector: ", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
@ -1,102 +0,0 @@
|
||||
/*
|
||||
* This file is part of the L2J Mobius project.
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package org.l2jmobius.commons.util;
|
||||
|
||||
import java.lang.management.ManagementFactory;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.logging.Logger;
|
||||
|
||||
/**
|
||||
* Thread to check for deadlocked threads.
|
||||
* @author -Nemesiss- L2M
|
||||
*/
|
||||
public class DeadlockDetector implements Runnable
|
||||
{
|
||||
protected static final Logger LOGGER = Logger.getLogger(DeadlockDetector.class.getName());
|
||||
private final Set<Long> _logged = new HashSet<>();
|
||||
|
||||
private static DeadlockDetector _instance;
|
||||
|
||||
public static DeadlockDetector getInstance()
|
||||
{
|
||||
if (_instance == null)
|
||||
{
|
||||
_instance = new DeadlockDetector();
|
||||
}
|
||||
return _instance;
|
||||
}
|
||||
|
||||
private DeadlockDetector()
|
||||
{
|
||||
LOGGER.info("DeadlockDetector daemon started.");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run()
|
||||
{
|
||||
final long[] ids = findDeadlockedThreadIDs();
|
||||
if (ids == null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
final List<Thread> deadlocked = new ArrayList<>();
|
||||
for (long id : ids)
|
||||
{
|
||||
if (_logged.add(id))
|
||||
{
|
||||
deadlocked.add(findThreadById(id));
|
||||
}
|
||||
}
|
||||
|
||||
if (!deadlocked.isEmpty())
|
||||
{
|
||||
Util.printSection("Deadlocked Thread(s)");
|
||||
for (Thread thread : deadlocked)
|
||||
{
|
||||
thread.getName();
|
||||
}
|
||||
|
||||
Util.printSection("End");
|
||||
}
|
||||
}
|
||||
|
||||
private long[] findDeadlockedThreadIDs()
|
||||
{
|
||||
if (ManagementFactory.getThreadMXBean().isSynchronizerUsageSupported())
|
||||
{
|
||||
return ManagementFactory.getThreadMXBean().findDeadlockedThreads();
|
||||
}
|
||||
return ManagementFactory.getThreadMXBean().findMonitorDeadlockedThreads();
|
||||
}
|
||||
|
||||
private Thread findThreadById(long id)
|
||||
{
|
||||
for (Thread thread : Thread.getAllStackTraces().keySet())
|
||||
{
|
||||
if (thread.getId() == id)
|
||||
{
|
||||
return thread;
|
||||
}
|
||||
}
|
||||
|
||||
throw new IllegalStateException("Deadlocked Thread not found!");
|
||||
}
|
||||
}
|
@ -21,6 +21,7 @@ import java.awt.Toolkit;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.InputStream;
|
||||
import java.time.Duration;
|
||||
import java.util.Calendar;
|
||||
import java.util.logging.LogManager;
|
||||
import java.util.logging.Logger;
|
||||
@ -30,7 +31,7 @@ import org.l2jmobius.commons.concurrent.ThreadPool;
|
||||
import org.l2jmobius.commons.database.DatabaseFactory;
|
||||
import org.l2jmobius.commons.enums.ServerMode;
|
||||
import org.l2jmobius.commons.util.Chronos;
|
||||
import org.l2jmobius.commons.util.DeadlockDetector;
|
||||
import org.l2jmobius.commons.util.DeadLockDetector;
|
||||
import org.l2jmobius.commons.util.PropertiesParser;
|
||||
import org.l2jmobius.commons.util.Util;
|
||||
import org.l2jmobius.gameserver.cache.CrestCache;
|
||||
@ -135,7 +136,6 @@ public class GameServer
|
||||
|
||||
private static TelnetStatusThread _statusServer;
|
||||
private static GameServer INSTANCE;
|
||||
|
||||
public static final Calendar dateTimeServerStarted = Calendar.getInstance();
|
||||
|
||||
public long getUsedMemoryMB()
|
||||
@ -175,10 +175,6 @@ public class GameServer
|
||||
|
||||
Util.printSection("ThreadPool");
|
||||
ThreadPool.init();
|
||||
if (Config.DEADLOCKCHECK_INTIAL_TIME > 0)
|
||||
{
|
||||
ThreadPool.scheduleAtFixedRate(DeadlockDetector.getInstance(), Config.DEADLOCKCHECK_INTIAL_TIME, Config.DEADLOCKCHECK_DELAY_TIME);
|
||||
}
|
||||
|
||||
Util.printSection("IdManager");
|
||||
IdManager.getInstance();
|
||||
@ -479,6 +475,19 @@ public class GameServer
|
||||
{
|
||||
PrecautionaryRestartManager.getInstance();
|
||||
}
|
||||
if (Config.DEADLOCK_DETECTOR)
|
||||
{
|
||||
final DeadLockDetector deadDetectThread = new DeadLockDetector(Duration.ofSeconds(Config.DEADLOCK_CHECK_INTERVAL), () ->
|
||||
{
|
||||
if (Config.RESTART_ON_DEADLOCK)
|
||||
{
|
||||
AnnouncementsTable.getInstance().announceToAll("Server has stability issues - restarting now.");
|
||||
Shutdown.getInstance().startShutdown(null, 60, true);
|
||||
}
|
||||
});
|
||||
deadDetectThread.setDaemon(true);
|
||||
deadDetectThread.start();
|
||||
}
|
||||
|
||||
Util.printSection("Status");
|
||||
|
||||
|
@ -130,6 +130,21 @@ InstantThreadPoolCount = 120
|
||||
UrgentPacketThreadCoreSize = 40
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dead Lock Detector (separate thread for detecting deadlocks)
|
||||
# ---------------------------------------------------------------------------
|
||||
# For improved crash logs and automatic restart in deadlock case if enabled.
|
||||
# Check interval is in seconds.
|
||||
# Default: True
|
||||
DeadLockDetector = True
|
||||
|
||||
# Default: 20
|
||||
DeadLockCheckInterval = 20
|
||||
|
||||
# Default: False
|
||||
RestartOnDeadlock = False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Misc Player Settings
|
||||
# ---------------------------------------------------------------------------
|
||||
|
@ -1,20 +0,0 @@
|
||||
# ---------------------------------------------------------------------------
|
||||
# Daemons settings
|
||||
# ---------------------------------------------------------------------------
|
||||
# Preamble:
|
||||
# Standard set up is possible only two parameters of demons:
|
||||
# - Pereodichnost run in milliseconds
|
||||
# - Delay in milliseconds before starting
|
||||
# Zero or the number of negative values of this trip of a parameter
|
||||
# For example:
|
||||
# DeadLockCheck = -1
|
||||
# Now the demon-stream after completing his tasks will disappear forever.
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dead Lock Detector (separate thread for detecting deadlocks)
|
||||
# ---------------------------------------------------------------------------
|
||||
# For improved crash logs and automatic restart in deadlock case if enabled.
|
||||
# Check interval is in seconds.
|
||||
# Default: 0 (disabled)
|
||||
DeadLockCheck = 0
|
||||
DeadLockDelay = 0
|
@ -91,7 +91,6 @@ public class Config
|
||||
private static final String SEVENSIGNS_CONFIG_FILE = "./config/main/SevenSigns.ini";
|
||||
public static final String SIEGE_CONFIG_FILE = "./config/main/Siege.ini";
|
||||
// protected
|
||||
private static final String DAEMONS_CONFIG_FILE = "./config/protected/Daemons.ini";
|
||||
private static final String PROTECT_FLOOD_CONFIG_FILE = "./config/protected/Flood.ini";
|
||||
private static final String PROTECT_OTHER_CONFIG_FILE = "./config/protected/Other.ini";
|
||||
public static final String TELNET_CONFIG_FILE = "./config/protected/Telnet.ini";
|
||||
@ -1139,9 +1138,6 @@ public class Config
|
||||
public static boolean ALLOW_REMOTE_CLASS_MASTERS;
|
||||
public static boolean ENABLE_EXP_GAIN_COMMANDS;
|
||||
|
||||
public static long DEADLOCKCHECK_INTIAL_TIME;
|
||||
public static long DEADLOCKCHECK_DELAY_TIME;
|
||||
|
||||
public static List<String> QUESTION_LIST = new ArrayList<>();
|
||||
|
||||
public static int SERVER_ID;
|
||||
@ -1169,6 +1165,9 @@ public class Config
|
||||
public static int SCHEDULED_THREAD_POOL_COUNT;
|
||||
public static int INSTANT_THREAD_POOL_COUNT;
|
||||
public static int IO_PACKET_THREAD_CORE_SIZE;
|
||||
public static boolean DEADLOCK_DETECTOR;
|
||||
public static int DEADLOCK_CHECK_INTERVAL;
|
||||
public static boolean RESTART_ON_DEADLOCK;
|
||||
public static String CNAME_TEMPLATE;
|
||||
public static String PET_NAME_TEMPLATE;
|
||||
public static String CLAN_NAME_TEMPLATE;
|
||||
@ -1255,6 +1254,9 @@ public class Config
|
||||
SCHEDULED_THREAD_POOL_COUNT = serverConfig.getInt("ScheduledThreadPoolCount", 40);
|
||||
INSTANT_THREAD_POOL_COUNT = serverConfig.getInt("InstantThreadPoolCount", 20);
|
||||
IO_PACKET_THREAD_CORE_SIZE = serverConfig.getInt("UrgentPacketThreadCoreSize", 20);
|
||||
DEADLOCK_DETECTOR = serverConfig.getBoolean("DeadLockDetector", true);
|
||||
DEADLOCK_CHECK_INTERVAL = serverConfig.getInt("DeadLockCheckInterval", 20);
|
||||
RESTART_ON_DEADLOCK = serverConfig.getBoolean("RestartOnDeadlock", false);
|
||||
CNAME_TEMPLATE = serverConfig.getString("CnameTemplate", ".*");
|
||||
PET_NAME_TEMPLATE = serverConfig.getString("PetNameTemplate", ".*");
|
||||
CLAN_NAME_TEMPLATE = serverConfig.getString("ClanNameTemplate", ".*");
|
||||
@ -2918,13 +2920,6 @@ public class Config
|
||||
ENABLE_EXP_GAIN_COMMANDS = characterConfig.getBoolean("EnableExpGainCommands", false);
|
||||
}
|
||||
|
||||
public static void loadDaemonsConf()
|
||||
{
|
||||
final PropertiesParser deamonsConfig = new PropertiesParser(DAEMONS_CONFIG_FILE);
|
||||
DEADLOCKCHECK_INTIAL_TIME = deamonsConfig.getLong("DeadLockCheck", 0);
|
||||
DEADLOCKCHECK_DELAY_TIME = deamonsConfig.getLong("DeadLockDelay", 0);
|
||||
}
|
||||
|
||||
/**
|
||||
* Loads all Filter Words
|
||||
*/
|
||||
@ -3138,9 +3133,6 @@ public class Config
|
||||
loadPCBPointConfig();
|
||||
loadOfflineConfig();
|
||||
|
||||
// Other
|
||||
loadDaemonsConf();
|
||||
|
||||
if (USE_SAY_FILTER)
|
||||
{
|
||||
loadFilter();
|
||||
|
@ -0,0 +1,120 @@
|
||||
/*
|
||||
* This file is part of the L2J Mobius project.
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package org.l2jmobius.commons.util;
|
||||
|
||||
import java.lang.management.LockInfo;
|
||||
import java.lang.management.ManagementFactory;
|
||||
import java.lang.management.MonitorInfo;
|
||||
import java.lang.management.ThreadInfo;
|
||||
import java.lang.management.ThreadMXBean;
|
||||
import java.time.Duration;
|
||||
import java.util.logging.Level;
|
||||
import java.util.logging.Logger;
|
||||
|
||||
import org.l2jmobius.Config;
|
||||
|
||||
/**
|
||||
* Thread to check for deadlocked threads.
|
||||
* @author -Nemesiss- L2M
|
||||
*/
|
||||
public class DeadLockDetector extends Thread
|
||||
{
|
||||
private static final Logger LOGGER = Logger.getLogger(DeadLockDetector.class.getName());
|
||||
|
||||
private final Duration _checkInterval;
|
||||
private final Runnable _deadLockCallback;
|
||||
private final ThreadMXBean tmx;
|
||||
|
||||
public DeadLockDetector(Duration checkInterval, Runnable deadLockCallback)
|
||||
{
|
||||
super("DeadLockDetector");
|
||||
_checkInterval = checkInterval;
|
||||
_deadLockCallback = deadLockCallback;
|
||||
tmx = ManagementFactory.getThreadMXBean();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run()
|
||||
{
|
||||
boolean deadlock = false;
|
||||
while (!deadlock)
|
||||
{
|
||||
try
|
||||
{
|
||||
final long[] ids = tmx.findDeadlockedThreads();
|
||||
if (ids != null)
|
||||
{
|
||||
deadlock = true;
|
||||
final ThreadInfo[] tis = tmx.getThreadInfo(ids, true, true);
|
||||
final StringBuilder info = new StringBuilder();
|
||||
info.append("DeadLock Found!");
|
||||
info.append(Config.EOL);
|
||||
for (ThreadInfo ti : tis)
|
||||
{
|
||||
info.append(ti.toString());
|
||||
}
|
||||
|
||||
for (ThreadInfo ti : tis)
|
||||
{
|
||||
final LockInfo[] locks = ti.getLockedSynchronizers();
|
||||
final MonitorInfo[] monitors = ti.getLockedMonitors();
|
||||
if ((locks.length == 0) && (monitors.length == 0))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
ThreadInfo dl = ti;
|
||||
info.append("Java-level deadlock:");
|
||||
info.append(Config.EOL);
|
||||
info.append('\t');
|
||||
info.append(dl.getThreadName());
|
||||
info.append(" is waiting to lock ");
|
||||
info.append(dl.getLockInfo().toString());
|
||||
info.append(" which is held by ");
|
||||
info.append(dl.getLockOwnerName());
|
||||
info.append(Config.EOL);
|
||||
while ((dl = tmx.getThreadInfo(new long[]
|
||||
{
|
||||
dl.getLockOwnerId()
|
||||
}, true, true)[0]).getThreadId() != ti.getThreadId())
|
||||
{
|
||||
info.append('\t');
|
||||
info.append(dl.getThreadName());
|
||||
info.append(" is waiting to lock ");
|
||||
info.append(dl.getLockInfo().toString());
|
||||
info.append(" which is held by ");
|
||||
info.append(dl.getLockOwnerName());
|
||||
info.append(Config.EOL);
|
||||
}
|
||||
}
|
||||
|
||||
LOGGER.warning(info.toString());
|
||||
|
||||
if (_deadLockCallback != null)
|
||||
{
|
||||
_deadLockCallback.run();
|
||||
}
|
||||
}
|
||||
Thread.sleep(_checkInterval.toMillis());
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
LOGGER.log(Level.WARNING, "DeadLockDetector: ", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
@ -1,102 +0,0 @@
|
||||
/*
|
||||
* This file is part of the L2J Mobius project.
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
* General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package org.l2jmobius.commons.util;
|
||||
|
||||
import java.lang.management.ManagementFactory;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Set;
|
||||
import java.util.logging.Logger;
|
||||
|
||||
/**
|
||||
* Thread to check for deadlocked threads.
|
||||
* @author -Nemesiss- L2M
|
||||
*/
|
||||
public class DeadlockDetector implements Runnable
|
||||
{
|
||||
protected static final Logger LOGGER = Logger.getLogger(DeadlockDetector.class.getName());
|
||||
private final Set<Long> _logged = new HashSet<>();
|
||||
|
||||
private static DeadlockDetector _instance;
|
||||
|
||||
public static DeadlockDetector getInstance()
|
||||
{
|
||||
if (_instance == null)
|
||||
{
|
||||
_instance = new DeadlockDetector();
|
||||
}
|
||||
return _instance;
|
||||
}
|
||||
|
||||
private DeadlockDetector()
|
||||
{
|
||||
LOGGER.info("DeadlockDetector daemon started.");
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run()
|
||||
{
|
||||
final long[] ids = findDeadlockedThreadIDs();
|
||||
if (ids == null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
final List<Thread> deadlocked = new ArrayList<>();
|
||||
for (long id : ids)
|
||||
{
|
||||
if (_logged.add(id))
|
||||
{
|
||||
deadlocked.add(findThreadById(id));
|
||||
}
|
||||
}
|
||||
|
||||
if (!deadlocked.isEmpty())
|
||||
{
|
||||
Util.printSection("Deadlocked Thread(s)");
|
||||
for (Thread thread : deadlocked)
|
||||
{
|
||||
thread.getName();
|
||||
}
|
||||
|
||||
Util.printSection("End");
|
||||
}
|
||||
}
|
||||
|
||||
private long[] findDeadlockedThreadIDs()
|
||||
{
|
||||
if (ManagementFactory.getThreadMXBean().isSynchronizerUsageSupported())
|
||||
{
|
||||
return ManagementFactory.getThreadMXBean().findDeadlockedThreads();
|
||||
}
|
||||
return ManagementFactory.getThreadMXBean().findMonitorDeadlockedThreads();
|
||||
}
|
||||
|
||||
private Thread findThreadById(long id)
|
||||
{
|
||||
for (Thread thread : Thread.getAllStackTraces().keySet())
|
||||
{
|
||||
if (thread.getId() == id)
|
||||
{
|
||||
return thread;
|
||||
}
|
||||
}
|
||||
|
||||
throw new IllegalStateException("Deadlocked Thread not found!");
|
||||
}
|
||||
}
|
@ -21,6 +21,7 @@ import java.awt.Toolkit;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.InputStream;
|
||||
import java.time.Duration;
|
||||
import java.util.Calendar;
|
||||
import java.util.logging.LogManager;
|
||||
import java.util.logging.Logger;
|
||||
@ -30,7 +31,7 @@ import org.l2jmobius.commons.concurrent.ThreadPool;
|
||||
import org.l2jmobius.commons.database.DatabaseFactory;
|
||||
import org.l2jmobius.commons.enums.ServerMode;
|
||||
import org.l2jmobius.commons.util.Chronos;
|
||||
import org.l2jmobius.commons.util.DeadlockDetector;
|
||||
import org.l2jmobius.commons.util.DeadLockDetector;
|
||||
import org.l2jmobius.commons.util.PropertiesParser;
|
||||
import org.l2jmobius.commons.util.Util;
|
||||
import org.l2jmobius.gameserver.cache.CrestCache;
|
||||
@ -139,7 +140,6 @@ public class GameServer
|
||||
|
||||
private static TelnetStatusThread _statusServer;
|
||||
private static GameServer INSTANCE;
|
||||
|
||||
public static final Calendar dateTimeServerStarted = Calendar.getInstance();
|
||||
|
||||
public long getUsedMemoryMB()
|
||||
@ -179,10 +179,6 @@ public class GameServer
|
||||
|
||||
Util.printSection("ThreadPool");
|
||||
ThreadPool.init();
|
||||
if (Config.DEADLOCKCHECK_INTIAL_TIME > 0)
|
||||
{
|
||||
ThreadPool.scheduleAtFixedRate(DeadlockDetector.getInstance(), Config.DEADLOCKCHECK_INTIAL_TIME, Config.DEADLOCKCHECK_DELAY_TIME);
|
||||
}
|
||||
|
||||
Util.printSection("IdManager");
|
||||
IdManager.getInstance();
|
||||
@ -491,6 +487,19 @@ public class GameServer
|
||||
{
|
||||
PrecautionaryRestartManager.getInstance();
|
||||
}
|
||||
if (Config.DEADLOCK_DETECTOR)
|
||||
{
|
||||
final DeadLockDetector deadDetectThread = new DeadLockDetector(Duration.ofSeconds(Config.DEADLOCK_CHECK_INTERVAL), () ->
|
||||
{
|
||||
if (Config.RESTART_ON_DEADLOCK)
|
||||
{
|
||||
AnnouncementsTable.getInstance().announceToAll("Server has stability issues - restarting now.");
|
||||
Shutdown.getInstance().startShutdown(null, 60, true);
|
||||
}
|
||||
});
|
||||
deadDetectThread.setDaemon(true);
|
||||
deadDetectThread.start();
|
||||
}
|
||||
|
||||
Util.printSection("Status");
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user