Updated deadlock detector.
This commit is contained in:
parent
ee84ae87fa
commit
6b5c3c6d21
@ -134,6 +134,21 @@ InstantThreadPoolCount = 120
|
|||||||
UrgentPacketThreadCoreSize = 40
|
UrgentPacketThreadCoreSize = 40
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Dead Lock Detector (separate thread for detecting deadlocks)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# For improved crash logs and automatic restart in deadlock case if enabled.
|
||||||
|
# Check interval is in seconds.
|
||||||
|
# Default: True
|
||||||
|
DeadLockDetector = True
|
||||||
|
|
||||||
|
# Default: 20
|
||||||
|
DeadLockCheckInterval = 20
|
||||||
|
|
||||||
|
# Default: False
|
||||||
|
RestartOnDeadlock = False
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Misc Player Settings
|
# Misc Player Settings
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
@ -1,20 +0,0 @@
|
|||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Daemons settings
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Preamble:
|
|
||||||
# Standard set up is possible only two parameters of demons:
|
|
||||||
# - Pereodichnost run in milliseconds
|
|
||||||
# - Delay in milliseconds before starting
|
|
||||||
# Zero or the number of negative values of this trip of a parameter
|
|
||||||
# For example:
|
|
||||||
# DeadLockCheck = -1
|
|
||||||
# Now the demon-stream after completing his tasks will disappear forever.
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Dead Lock Detector (separate thread for detecting deadlocks)
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# For improved crash logs and automatic restart in deadlock case if enabled.
|
|
||||||
# Check interval is in seconds.
|
|
||||||
# Default: 0 (disabled)
|
|
||||||
DeadLockCheck = 0
|
|
||||||
DeadLockDelay = 0
|
|
@ -80,7 +80,6 @@ public class Config
|
|||||||
private static final String SEVENSIGNS_CONFIG_FILE = "./config/main/SevenSigns.ini";
|
private static final String SEVENSIGNS_CONFIG_FILE = "./config/main/SevenSigns.ini";
|
||||||
public static final String SIEGE_CONFIG_FILE = "./config/main/Siege.ini";
|
public static final String SIEGE_CONFIG_FILE = "./config/main/Siege.ini";
|
||||||
// protected
|
// protected
|
||||||
private static final String DAEMONS_CONFIG_FILE = "./config/protected/Daemons.ini";
|
|
||||||
private static final String PROTECT_FLOOD_CONFIG_FILE = "./config/protected/Flood.ini";
|
private static final String PROTECT_FLOOD_CONFIG_FILE = "./config/protected/Flood.ini";
|
||||||
private static final String PROTECT_OTHER_CONFIG_FILE = "./config/protected/Other.ini";
|
private static final String PROTECT_OTHER_CONFIG_FILE = "./config/protected/Other.ini";
|
||||||
public static final String TELNET_CONFIG_FILE = "./config/protected/Telnet.ini";
|
public static final String TELNET_CONFIG_FILE = "./config/protected/Telnet.ini";
|
||||||
@ -1102,9 +1101,6 @@ public class Config
|
|||||||
public static boolean ALLOW_REMOTE_CLASS_MASTERS;
|
public static boolean ALLOW_REMOTE_CLASS_MASTERS;
|
||||||
public static boolean ENABLE_EXP_GAIN_COMMANDS;
|
public static boolean ENABLE_EXP_GAIN_COMMANDS;
|
||||||
|
|
||||||
public static long DEADLOCKCHECK_INTIAL_TIME;
|
|
||||||
public static long DEADLOCKCHECK_DELAY_TIME;
|
|
||||||
|
|
||||||
public static List<String> QUESTION_LIST = new ArrayList<>();
|
public static List<String> QUESTION_LIST = new ArrayList<>();
|
||||||
|
|
||||||
public static int SERVER_ID;
|
public static int SERVER_ID;
|
||||||
@ -1130,6 +1126,9 @@ public class Config
|
|||||||
public static int SCHEDULED_THREAD_POOL_COUNT;
|
public static int SCHEDULED_THREAD_POOL_COUNT;
|
||||||
public static int INSTANT_THREAD_POOL_COUNT;
|
public static int INSTANT_THREAD_POOL_COUNT;
|
||||||
public static int IO_PACKET_THREAD_CORE_SIZE;
|
public static int IO_PACKET_THREAD_CORE_SIZE;
|
||||||
|
public static boolean DEADLOCK_DETECTOR;
|
||||||
|
public static int DEADLOCK_CHECK_INTERVAL;
|
||||||
|
public static boolean RESTART_ON_DEADLOCK;
|
||||||
public static String CNAME_TEMPLATE;
|
public static String CNAME_TEMPLATE;
|
||||||
public static String PET_NAME_TEMPLATE;
|
public static String PET_NAME_TEMPLATE;
|
||||||
public static String CLAN_NAME_TEMPLATE;
|
public static String CLAN_NAME_TEMPLATE;
|
||||||
@ -1222,6 +1221,9 @@ public class Config
|
|||||||
SCHEDULED_THREAD_POOL_COUNT = serverConfig.getInt("ScheduledThreadPoolCount", 40);
|
SCHEDULED_THREAD_POOL_COUNT = serverConfig.getInt("ScheduledThreadPoolCount", 40);
|
||||||
INSTANT_THREAD_POOL_COUNT = serverConfig.getInt("InstantThreadPoolCount", 20);
|
INSTANT_THREAD_POOL_COUNT = serverConfig.getInt("InstantThreadPoolCount", 20);
|
||||||
IO_PACKET_THREAD_CORE_SIZE = serverConfig.getInt("UrgentPacketThreadCoreSize", 20);
|
IO_PACKET_THREAD_CORE_SIZE = serverConfig.getInt("UrgentPacketThreadCoreSize", 20);
|
||||||
|
DEADLOCK_DETECTOR = serverConfig.getBoolean("DeadLockDetector", true);
|
||||||
|
DEADLOCK_CHECK_INTERVAL = serverConfig.getInt("DeadLockCheckInterval", 20);
|
||||||
|
RESTART_ON_DEADLOCK = serverConfig.getBoolean("RestartOnDeadlock", false);
|
||||||
CNAME_TEMPLATE = serverConfig.getString("CnameTemplate", ".*");
|
CNAME_TEMPLATE = serverConfig.getString("CnameTemplate", ".*");
|
||||||
PET_NAME_TEMPLATE = serverConfig.getString("PetNameTemplate", ".*");
|
PET_NAME_TEMPLATE = serverConfig.getString("PetNameTemplate", ".*");
|
||||||
CLAN_NAME_TEMPLATE = serverConfig.getString("ClanNameTemplate", ".*");
|
CLAN_NAME_TEMPLATE = serverConfig.getString("ClanNameTemplate", ".*");
|
||||||
@ -2854,13 +2856,6 @@ public class Config
|
|||||||
ENABLE_EXP_GAIN_COMMANDS = characterConfig.getBoolean("EnableExpGainCommands", false);
|
ENABLE_EXP_GAIN_COMMANDS = characterConfig.getBoolean("EnableExpGainCommands", false);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static void loadDaemonsConf()
|
|
||||||
{
|
|
||||||
final PropertiesParser deamonsConfig = new PropertiesParser(DAEMONS_CONFIG_FILE);
|
|
||||||
DEADLOCKCHECK_INTIAL_TIME = deamonsConfig.getLong("DeadLockCheck", 0);
|
|
||||||
DEADLOCKCHECK_DELAY_TIME = deamonsConfig.getLong("DeadLockDelay", 0);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Loads all Filter Words
|
* Loads all Filter Words
|
||||||
*/
|
*/
|
||||||
@ -3158,9 +3153,6 @@ public class Config
|
|||||||
loadPCBPointConfig();
|
loadPCBPointConfig();
|
||||||
loadOfflineConfig();
|
loadOfflineConfig();
|
||||||
|
|
||||||
// Other
|
|
||||||
loadDaemonsConf();
|
|
||||||
|
|
||||||
if (USE_SAY_FILTER)
|
if (USE_SAY_FILTER)
|
||||||
{
|
{
|
||||||
loadFilter();
|
loadFilter();
|
||||||
|
@ -0,0 +1,120 @@
|
|||||||
|
/*
|
||||||
|
* This file is part of the L2J Mobius project.
|
||||||
|
*
|
||||||
|
* This program is free software: you can redistribute it and/or modify
|
||||||
|
* it under the terms of the GNU General Public License as published by
|
||||||
|
* the Free Software Foundation, either version 3 of the License, or
|
||||||
|
* (at your option) any later version.
|
||||||
|
*
|
||||||
|
* This program is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||||
|
* General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU General Public License
|
||||||
|
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
*/
|
||||||
|
package org.l2jmobius.commons.util;
|
||||||
|
|
||||||
|
import java.lang.management.LockInfo;
|
||||||
|
import java.lang.management.ManagementFactory;
|
||||||
|
import java.lang.management.MonitorInfo;
|
||||||
|
import java.lang.management.ThreadInfo;
|
||||||
|
import java.lang.management.ThreadMXBean;
|
||||||
|
import java.time.Duration;
|
||||||
|
import java.util.logging.Level;
|
||||||
|
import java.util.logging.Logger;
|
||||||
|
|
||||||
|
import org.l2jmobius.Config;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Thread to check for deadlocked threads.
|
||||||
|
* @author -Nemesiss- L2M
|
||||||
|
*/
|
||||||
|
public class DeadLockDetector extends Thread
|
||||||
|
{
|
||||||
|
private static final Logger LOGGER = Logger.getLogger(DeadLockDetector.class.getName());
|
||||||
|
|
||||||
|
private final Duration _checkInterval;
|
||||||
|
private final Runnable _deadLockCallback;
|
||||||
|
private final ThreadMXBean tmx;
|
||||||
|
|
||||||
|
public DeadLockDetector(Duration checkInterval, Runnable deadLockCallback)
|
||||||
|
{
|
||||||
|
super("DeadLockDetector");
|
||||||
|
_checkInterval = checkInterval;
|
||||||
|
_deadLockCallback = deadLockCallback;
|
||||||
|
tmx = ManagementFactory.getThreadMXBean();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void run()
|
||||||
|
{
|
||||||
|
boolean deadlock = false;
|
||||||
|
while (!deadlock)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
final long[] ids = tmx.findDeadlockedThreads();
|
||||||
|
if (ids != null)
|
||||||
|
{
|
||||||
|
deadlock = true;
|
||||||
|
final ThreadInfo[] tis = tmx.getThreadInfo(ids, true, true);
|
||||||
|
final StringBuilder info = new StringBuilder();
|
||||||
|
info.append("DeadLock Found!");
|
||||||
|
info.append(Config.EOL);
|
||||||
|
for (ThreadInfo ti : tis)
|
||||||
|
{
|
||||||
|
info.append(ti.toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
for (ThreadInfo ti : tis)
|
||||||
|
{
|
||||||
|
final LockInfo[] locks = ti.getLockedSynchronizers();
|
||||||
|
final MonitorInfo[] monitors = ti.getLockedMonitors();
|
||||||
|
if ((locks.length == 0) && (monitors.length == 0))
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
ThreadInfo dl = ti;
|
||||||
|
info.append("Java-level deadlock:");
|
||||||
|
info.append(Config.EOL);
|
||||||
|
info.append('\t');
|
||||||
|
info.append(dl.getThreadName());
|
||||||
|
info.append(" is waiting to lock ");
|
||||||
|
info.append(dl.getLockInfo().toString());
|
||||||
|
info.append(" which is held by ");
|
||||||
|
info.append(dl.getLockOwnerName());
|
||||||
|
info.append(Config.EOL);
|
||||||
|
while ((dl = tmx.getThreadInfo(new long[]
|
||||||
|
{
|
||||||
|
dl.getLockOwnerId()
|
||||||
|
}, true, true)[0]).getThreadId() != ti.getThreadId())
|
||||||
|
{
|
||||||
|
info.append('\t');
|
||||||
|
info.append(dl.getThreadName());
|
||||||
|
info.append(" is waiting to lock ");
|
||||||
|
info.append(dl.getLockInfo().toString());
|
||||||
|
info.append(" which is held by ");
|
||||||
|
info.append(dl.getLockOwnerName());
|
||||||
|
info.append(Config.EOL);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
LOGGER.warning(info.toString());
|
||||||
|
|
||||||
|
if (_deadLockCallback != null)
|
||||||
|
{
|
||||||
|
_deadLockCallback.run();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Thread.sleep(_checkInterval.toMillis());
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
|
{
|
||||||
|
LOGGER.log(Level.WARNING, "DeadLockDetector: ", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@ -1,102 +0,0 @@
|
|||||||
/*
|
|
||||||
* This file is part of the L2J Mobius project.
|
|
||||||
*
|
|
||||||
* This program is free software: you can redistribute it and/or modify
|
|
||||||
* it under the terms of the GNU General Public License as published by
|
|
||||||
* the Free Software Foundation, either version 3 of the License, or
|
|
||||||
* (at your option) any later version.
|
|
||||||
*
|
|
||||||
* This program is distributed in the hope that it will be useful,
|
|
||||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
||||||
* General Public License for more details.
|
|
||||||
*
|
|
||||||
* You should have received a copy of the GNU General Public License
|
|
||||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
||||||
*/
|
|
||||||
package org.l2jmobius.commons.util;
|
|
||||||
|
|
||||||
import java.lang.management.ManagementFactory;
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.HashSet;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Set;
|
|
||||||
import java.util.logging.Logger;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Thread to check for deadlocked threads.
|
|
||||||
* @author -Nemesiss- L2M
|
|
||||||
*/
|
|
||||||
public class DeadlockDetector implements Runnable
|
|
||||||
{
|
|
||||||
protected static final Logger LOGGER = Logger.getLogger(DeadlockDetector.class.getName());
|
|
||||||
private final Set<Long> _logged = new HashSet<>();
|
|
||||||
|
|
||||||
private static DeadlockDetector _instance;
|
|
||||||
|
|
||||||
public static DeadlockDetector getInstance()
|
|
||||||
{
|
|
||||||
if (_instance == null)
|
|
||||||
{
|
|
||||||
_instance = new DeadlockDetector();
|
|
||||||
}
|
|
||||||
return _instance;
|
|
||||||
}
|
|
||||||
|
|
||||||
private DeadlockDetector()
|
|
||||||
{
|
|
||||||
LOGGER.info("DeadlockDetector daemon started.");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void run()
|
|
||||||
{
|
|
||||||
final long[] ids = findDeadlockedThreadIDs();
|
|
||||||
if (ids == null)
|
|
||||||
{
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
final List<Thread> deadlocked = new ArrayList<>();
|
|
||||||
for (long id : ids)
|
|
||||||
{
|
|
||||||
if (_logged.add(id))
|
|
||||||
{
|
|
||||||
deadlocked.add(findThreadById(id));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!deadlocked.isEmpty())
|
|
||||||
{
|
|
||||||
Util.printSection("Deadlocked Thread(s)");
|
|
||||||
for (Thread thread : deadlocked)
|
|
||||||
{
|
|
||||||
thread.getName();
|
|
||||||
}
|
|
||||||
|
|
||||||
Util.printSection("End");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private long[] findDeadlockedThreadIDs()
|
|
||||||
{
|
|
||||||
if (ManagementFactory.getThreadMXBean().isSynchronizerUsageSupported())
|
|
||||||
{
|
|
||||||
return ManagementFactory.getThreadMXBean().findDeadlockedThreads();
|
|
||||||
}
|
|
||||||
return ManagementFactory.getThreadMXBean().findMonitorDeadlockedThreads();
|
|
||||||
}
|
|
||||||
|
|
||||||
private Thread findThreadById(long id)
|
|
||||||
{
|
|
||||||
for (Thread thread : Thread.getAllStackTraces().keySet())
|
|
||||||
{
|
|
||||||
if (thread.getId() == id)
|
|
||||||
{
|
|
||||||
return thread;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
throw new IllegalStateException("Deadlocked Thread not found!");
|
|
||||||
}
|
|
||||||
}
|
|
@ -21,6 +21,7 @@ import java.awt.Toolkit;
|
|||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.io.FileInputStream;
|
import java.io.FileInputStream;
|
||||||
import java.io.InputStream;
|
import java.io.InputStream;
|
||||||
|
import java.time.Duration;
|
||||||
import java.util.Calendar;
|
import java.util.Calendar;
|
||||||
import java.util.logging.LogManager;
|
import java.util.logging.LogManager;
|
||||||
import java.util.logging.Logger;
|
import java.util.logging.Logger;
|
||||||
@ -30,7 +31,7 @@ import org.l2jmobius.commons.concurrent.ThreadPool;
|
|||||||
import org.l2jmobius.commons.database.DatabaseFactory;
|
import org.l2jmobius.commons.database.DatabaseFactory;
|
||||||
import org.l2jmobius.commons.enums.ServerMode;
|
import org.l2jmobius.commons.enums.ServerMode;
|
||||||
import org.l2jmobius.commons.util.Chronos;
|
import org.l2jmobius.commons.util.Chronos;
|
||||||
import org.l2jmobius.commons.util.DeadlockDetector;
|
import org.l2jmobius.commons.util.DeadLockDetector;
|
||||||
import org.l2jmobius.commons.util.PropertiesParser;
|
import org.l2jmobius.commons.util.PropertiesParser;
|
||||||
import org.l2jmobius.commons.util.Util;
|
import org.l2jmobius.commons.util.Util;
|
||||||
import org.l2jmobius.gameserver.cache.CrestCache;
|
import org.l2jmobius.gameserver.cache.CrestCache;
|
||||||
@ -135,7 +136,6 @@ public class GameServer
|
|||||||
|
|
||||||
private static TelnetStatusThread _statusServer;
|
private static TelnetStatusThread _statusServer;
|
||||||
private static GameServer INSTANCE;
|
private static GameServer INSTANCE;
|
||||||
|
|
||||||
public static final Calendar dateTimeServerStarted = Calendar.getInstance();
|
public static final Calendar dateTimeServerStarted = Calendar.getInstance();
|
||||||
|
|
||||||
public long getUsedMemoryMB()
|
public long getUsedMemoryMB()
|
||||||
@ -175,10 +175,6 @@ public class GameServer
|
|||||||
|
|
||||||
Util.printSection("ThreadPool");
|
Util.printSection("ThreadPool");
|
||||||
ThreadPool.init();
|
ThreadPool.init();
|
||||||
if (Config.DEADLOCKCHECK_INTIAL_TIME > 0)
|
|
||||||
{
|
|
||||||
ThreadPool.scheduleAtFixedRate(DeadlockDetector.getInstance(), Config.DEADLOCKCHECK_INTIAL_TIME, Config.DEADLOCKCHECK_DELAY_TIME);
|
|
||||||
}
|
|
||||||
|
|
||||||
Util.printSection("IdManager");
|
Util.printSection("IdManager");
|
||||||
IdManager.getInstance();
|
IdManager.getInstance();
|
||||||
@ -479,6 +475,19 @@ public class GameServer
|
|||||||
{
|
{
|
||||||
PrecautionaryRestartManager.getInstance();
|
PrecautionaryRestartManager.getInstance();
|
||||||
}
|
}
|
||||||
|
if (Config.DEADLOCK_DETECTOR)
|
||||||
|
{
|
||||||
|
final DeadLockDetector deadDetectThread = new DeadLockDetector(Duration.ofSeconds(Config.DEADLOCK_CHECK_INTERVAL), () ->
|
||||||
|
{
|
||||||
|
if (Config.RESTART_ON_DEADLOCK)
|
||||||
|
{
|
||||||
|
AnnouncementsTable.getInstance().announceToAll("Server has stability issues - restarting now.");
|
||||||
|
Shutdown.getInstance().startShutdown(null, 60, true);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
deadDetectThread.setDaemon(true);
|
||||||
|
deadDetectThread.start();
|
||||||
|
}
|
||||||
|
|
||||||
Util.printSection("Status");
|
Util.printSection("Status");
|
||||||
|
|
||||||
|
@ -130,6 +130,21 @@ InstantThreadPoolCount = 120
|
|||||||
UrgentPacketThreadCoreSize = 40
|
UrgentPacketThreadCoreSize = 40
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Dead Lock Detector (separate thread for detecting deadlocks)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# For improved crash logs and automatic restart in deadlock case if enabled.
|
||||||
|
# Check interval is in seconds.
|
||||||
|
# Default: True
|
||||||
|
DeadLockDetector = True
|
||||||
|
|
||||||
|
# Default: 20
|
||||||
|
DeadLockCheckInterval = 20
|
||||||
|
|
||||||
|
# Default: False
|
||||||
|
RestartOnDeadlock = False
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Misc Player Settings
|
# Misc Player Settings
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
|
@ -1,20 +0,0 @@
|
|||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Daemons settings
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Preamble:
|
|
||||||
# Standard set up is possible only two parameters of demons:
|
|
||||||
# - Pereodichnost run in milliseconds
|
|
||||||
# - Delay in milliseconds before starting
|
|
||||||
# Zero or the number of negative values of this trip of a parameter
|
|
||||||
# For example:
|
|
||||||
# DeadLockCheck = -1
|
|
||||||
# Now the demon-stream after completing his tasks will disappear forever.
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# Dead Lock Detector (separate thread for detecting deadlocks)
|
|
||||||
# ---------------------------------------------------------------------------
|
|
||||||
# For improved crash logs and automatic restart in deadlock case if enabled.
|
|
||||||
# Check interval is in seconds.
|
|
||||||
# Default: 0 (disabled)
|
|
||||||
DeadLockCheck = 0
|
|
||||||
DeadLockDelay = 0
|
|
@ -91,7 +91,6 @@ public class Config
|
|||||||
private static final String SEVENSIGNS_CONFIG_FILE = "./config/main/SevenSigns.ini";
|
private static final String SEVENSIGNS_CONFIG_FILE = "./config/main/SevenSigns.ini";
|
||||||
public static final String SIEGE_CONFIG_FILE = "./config/main/Siege.ini";
|
public static final String SIEGE_CONFIG_FILE = "./config/main/Siege.ini";
|
||||||
// protected
|
// protected
|
||||||
private static final String DAEMONS_CONFIG_FILE = "./config/protected/Daemons.ini";
|
|
||||||
private static final String PROTECT_FLOOD_CONFIG_FILE = "./config/protected/Flood.ini";
|
private static final String PROTECT_FLOOD_CONFIG_FILE = "./config/protected/Flood.ini";
|
||||||
private static final String PROTECT_OTHER_CONFIG_FILE = "./config/protected/Other.ini";
|
private static final String PROTECT_OTHER_CONFIG_FILE = "./config/protected/Other.ini";
|
||||||
public static final String TELNET_CONFIG_FILE = "./config/protected/Telnet.ini";
|
public static final String TELNET_CONFIG_FILE = "./config/protected/Telnet.ini";
|
||||||
@ -1139,9 +1138,6 @@ public class Config
|
|||||||
public static boolean ALLOW_REMOTE_CLASS_MASTERS;
|
public static boolean ALLOW_REMOTE_CLASS_MASTERS;
|
||||||
public static boolean ENABLE_EXP_GAIN_COMMANDS;
|
public static boolean ENABLE_EXP_GAIN_COMMANDS;
|
||||||
|
|
||||||
public static long DEADLOCKCHECK_INTIAL_TIME;
|
|
||||||
public static long DEADLOCKCHECK_DELAY_TIME;
|
|
||||||
|
|
||||||
public static List<String> QUESTION_LIST = new ArrayList<>();
|
public static List<String> QUESTION_LIST = new ArrayList<>();
|
||||||
|
|
||||||
public static int SERVER_ID;
|
public static int SERVER_ID;
|
||||||
@ -1169,6 +1165,9 @@ public class Config
|
|||||||
public static int SCHEDULED_THREAD_POOL_COUNT;
|
public static int SCHEDULED_THREAD_POOL_COUNT;
|
||||||
public static int INSTANT_THREAD_POOL_COUNT;
|
public static int INSTANT_THREAD_POOL_COUNT;
|
||||||
public static int IO_PACKET_THREAD_CORE_SIZE;
|
public static int IO_PACKET_THREAD_CORE_SIZE;
|
||||||
|
public static boolean DEADLOCK_DETECTOR;
|
||||||
|
public static int DEADLOCK_CHECK_INTERVAL;
|
||||||
|
public static boolean RESTART_ON_DEADLOCK;
|
||||||
public static String CNAME_TEMPLATE;
|
public static String CNAME_TEMPLATE;
|
||||||
public static String PET_NAME_TEMPLATE;
|
public static String PET_NAME_TEMPLATE;
|
||||||
public static String CLAN_NAME_TEMPLATE;
|
public static String CLAN_NAME_TEMPLATE;
|
||||||
@ -1255,6 +1254,9 @@ public class Config
|
|||||||
SCHEDULED_THREAD_POOL_COUNT = serverConfig.getInt("ScheduledThreadPoolCount", 40);
|
SCHEDULED_THREAD_POOL_COUNT = serverConfig.getInt("ScheduledThreadPoolCount", 40);
|
||||||
INSTANT_THREAD_POOL_COUNT = serverConfig.getInt("InstantThreadPoolCount", 20);
|
INSTANT_THREAD_POOL_COUNT = serverConfig.getInt("InstantThreadPoolCount", 20);
|
||||||
IO_PACKET_THREAD_CORE_SIZE = serverConfig.getInt("UrgentPacketThreadCoreSize", 20);
|
IO_PACKET_THREAD_CORE_SIZE = serverConfig.getInt("UrgentPacketThreadCoreSize", 20);
|
||||||
|
DEADLOCK_DETECTOR = serverConfig.getBoolean("DeadLockDetector", true);
|
||||||
|
DEADLOCK_CHECK_INTERVAL = serverConfig.getInt("DeadLockCheckInterval", 20);
|
||||||
|
RESTART_ON_DEADLOCK = serverConfig.getBoolean("RestartOnDeadlock", false);
|
||||||
CNAME_TEMPLATE = serverConfig.getString("CnameTemplate", ".*");
|
CNAME_TEMPLATE = serverConfig.getString("CnameTemplate", ".*");
|
||||||
PET_NAME_TEMPLATE = serverConfig.getString("PetNameTemplate", ".*");
|
PET_NAME_TEMPLATE = serverConfig.getString("PetNameTemplate", ".*");
|
||||||
CLAN_NAME_TEMPLATE = serverConfig.getString("ClanNameTemplate", ".*");
|
CLAN_NAME_TEMPLATE = serverConfig.getString("ClanNameTemplate", ".*");
|
||||||
@ -2918,13 +2920,6 @@ public class Config
|
|||||||
ENABLE_EXP_GAIN_COMMANDS = characterConfig.getBoolean("EnableExpGainCommands", false);
|
ENABLE_EXP_GAIN_COMMANDS = characterConfig.getBoolean("EnableExpGainCommands", false);
|
||||||
}
|
}
|
||||||
|
|
||||||
public static void loadDaemonsConf()
|
|
||||||
{
|
|
||||||
final PropertiesParser deamonsConfig = new PropertiesParser(DAEMONS_CONFIG_FILE);
|
|
||||||
DEADLOCKCHECK_INTIAL_TIME = deamonsConfig.getLong("DeadLockCheck", 0);
|
|
||||||
DEADLOCKCHECK_DELAY_TIME = deamonsConfig.getLong("DeadLockDelay", 0);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Loads all Filter Words
|
* Loads all Filter Words
|
||||||
*/
|
*/
|
||||||
@ -3138,9 +3133,6 @@ public class Config
|
|||||||
loadPCBPointConfig();
|
loadPCBPointConfig();
|
||||||
loadOfflineConfig();
|
loadOfflineConfig();
|
||||||
|
|
||||||
// Other
|
|
||||||
loadDaemonsConf();
|
|
||||||
|
|
||||||
if (USE_SAY_FILTER)
|
if (USE_SAY_FILTER)
|
||||||
{
|
{
|
||||||
loadFilter();
|
loadFilter();
|
||||||
|
@ -0,0 +1,120 @@
|
|||||||
|
/*
|
||||||
|
* This file is part of the L2J Mobius project.
|
||||||
|
*
|
||||||
|
* This program is free software: you can redistribute it and/or modify
|
||||||
|
* it under the terms of the GNU General Public License as published by
|
||||||
|
* the Free Software Foundation, either version 3 of the License, or
|
||||||
|
* (at your option) any later version.
|
||||||
|
*
|
||||||
|
* This program is distributed in the hope that it will be useful,
|
||||||
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||||
|
* General Public License for more details.
|
||||||
|
*
|
||||||
|
* You should have received a copy of the GNU General Public License
|
||||||
|
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
*/
|
||||||
|
package org.l2jmobius.commons.util;
|
||||||
|
|
||||||
|
import java.lang.management.LockInfo;
|
||||||
|
import java.lang.management.ManagementFactory;
|
||||||
|
import java.lang.management.MonitorInfo;
|
||||||
|
import java.lang.management.ThreadInfo;
|
||||||
|
import java.lang.management.ThreadMXBean;
|
||||||
|
import java.time.Duration;
|
||||||
|
import java.util.logging.Level;
|
||||||
|
import java.util.logging.Logger;
|
||||||
|
|
||||||
|
import org.l2jmobius.Config;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Thread to check for deadlocked threads.
|
||||||
|
* @author -Nemesiss- L2M
|
||||||
|
*/
|
||||||
|
public class DeadLockDetector extends Thread
|
||||||
|
{
|
||||||
|
private static final Logger LOGGER = Logger.getLogger(DeadLockDetector.class.getName());
|
||||||
|
|
||||||
|
private final Duration _checkInterval;
|
||||||
|
private final Runnable _deadLockCallback;
|
||||||
|
private final ThreadMXBean tmx;
|
||||||
|
|
||||||
|
public DeadLockDetector(Duration checkInterval, Runnable deadLockCallback)
|
||||||
|
{
|
||||||
|
super("DeadLockDetector");
|
||||||
|
_checkInterval = checkInterval;
|
||||||
|
_deadLockCallback = deadLockCallback;
|
||||||
|
tmx = ManagementFactory.getThreadMXBean();
|
||||||
|
}
|
||||||
|
|
||||||
|
@Override
|
||||||
|
public void run()
|
||||||
|
{
|
||||||
|
boolean deadlock = false;
|
||||||
|
while (!deadlock)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
final long[] ids = tmx.findDeadlockedThreads();
|
||||||
|
if (ids != null)
|
||||||
|
{
|
||||||
|
deadlock = true;
|
||||||
|
final ThreadInfo[] tis = tmx.getThreadInfo(ids, true, true);
|
||||||
|
final StringBuilder info = new StringBuilder();
|
||||||
|
info.append("DeadLock Found!");
|
||||||
|
info.append(Config.EOL);
|
||||||
|
for (ThreadInfo ti : tis)
|
||||||
|
{
|
||||||
|
info.append(ti.toString());
|
||||||
|
}
|
||||||
|
|
||||||
|
for (ThreadInfo ti : tis)
|
||||||
|
{
|
||||||
|
final LockInfo[] locks = ti.getLockedSynchronizers();
|
||||||
|
final MonitorInfo[] monitors = ti.getLockedMonitors();
|
||||||
|
if ((locks.length == 0) && (monitors.length == 0))
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
ThreadInfo dl = ti;
|
||||||
|
info.append("Java-level deadlock:");
|
||||||
|
info.append(Config.EOL);
|
||||||
|
info.append('\t');
|
||||||
|
info.append(dl.getThreadName());
|
||||||
|
info.append(" is waiting to lock ");
|
||||||
|
info.append(dl.getLockInfo().toString());
|
||||||
|
info.append(" which is held by ");
|
||||||
|
info.append(dl.getLockOwnerName());
|
||||||
|
info.append(Config.EOL);
|
||||||
|
while ((dl = tmx.getThreadInfo(new long[]
|
||||||
|
{
|
||||||
|
dl.getLockOwnerId()
|
||||||
|
}, true, true)[0]).getThreadId() != ti.getThreadId())
|
||||||
|
{
|
||||||
|
info.append('\t');
|
||||||
|
info.append(dl.getThreadName());
|
||||||
|
info.append(" is waiting to lock ");
|
||||||
|
info.append(dl.getLockInfo().toString());
|
||||||
|
info.append(" which is held by ");
|
||||||
|
info.append(dl.getLockOwnerName());
|
||||||
|
info.append(Config.EOL);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
LOGGER.warning(info.toString());
|
||||||
|
|
||||||
|
if (_deadLockCallback != null)
|
||||||
|
{
|
||||||
|
_deadLockCallback.run();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Thread.sleep(_checkInterval.toMillis());
|
||||||
|
}
|
||||||
|
catch (Exception e)
|
||||||
|
{
|
||||||
|
LOGGER.log(Level.WARNING, "DeadLockDetector: ", e);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
@ -1,102 +0,0 @@
|
|||||||
/*
|
|
||||||
* This file is part of the L2J Mobius project.
|
|
||||||
*
|
|
||||||
* This program is free software: you can redistribute it and/or modify
|
|
||||||
* it under the terms of the GNU General Public License as published by
|
|
||||||
* the Free Software Foundation, either version 3 of the License, or
|
|
||||||
* (at your option) any later version.
|
|
||||||
*
|
|
||||||
* This program is distributed in the hope that it will be useful,
|
|
||||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
||||||
* General Public License for more details.
|
|
||||||
*
|
|
||||||
* You should have received a copy of the GNU General Public License
|
|
||||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
||||||
*/
|
|
||||||
package org.l2jmobius.commons.util;
|
|
||||||
|
|
||||||
import java.lang.management.ManagementFactory;
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.HashSet;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Set;
|
|
||||||
import java.util.logging.Logger;
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Thread to check for deadlocked threads.
|
|
||||||
* @author -Nemesiss- L2M
|
|
||||||
*/
|
|
||||||
public class DeadlockDetector implements Runnable
|
|
||||||
{
|
|
||||||
protected static final Logger LOGGER = Logger.getLogger(DeadlockDetector.class.getName());
|
|
||||||
private final Set<Long> _logged = new HashSet<>();
|
|
||||||
|
|
||||||
private static DeadlockDetector _instance;
|
|
||||||
|
|
||||||
public static DeadlockDetector getInstance()
|
|
||||||
{
|
|
||||||
if (_instance == null)
|
|
||||||
{
|
|
||||||
_instance = new DeadlockDetector();
|
|
||||||
}
|
|
||||||
return _instance;
|
|
||||||
}
|
|
||||||
|
|
||||||
private DeadlockDetector()
|
|
||||||
{
|
|
||||||
LOGGER.info("DeadlockDetector daemon started.");
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void run()
|
|
||||||
{
|
|
||||||
final long[] ids = findDeadlockedThreadIDs();
|
|
||||||
if (ids == null)
|
|
||||||
{
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
final List<Thread> deadlocked = new ArrayList<>();
|
|
||||||
for (long id : ids)
|
|
||||||
{
|
|
||||||
if (_logged.add(id))
|
|
||||||
{
|
|
||||||
deadlocked.add(findThreadById(id));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!deadlocked.isEmpty())
|
|
||||||
{
|
|
||||||
Util.printSection("Deadlocked Thread(s)");
|
|
||||||
for (Thread thread : deadlocked)
|
|
||||||
{
|
|
||||||
thread.getName();
|
|
||||||
}
|
|
||||||
|
|
||||||
Util.printSection("End");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private long[] findDeadlockedThreadIDs()
|
|
||||||
{
|
|
||||||
if (ManagementFactory.getThreadMXBean().isSynchronizerUsageSupported())
|
|
||||||
{
|
|
||||||
return ManagementFactory.getThreadMXBean().findDeadlockedThreads();
|
|
||||||
}
|
|
||||||
return ManagementFactory.getThreadMXBean().findMonitorDeadlockedThreads();
|
|
||||||
}
|
|
||||||
|
|
||||||
private Thread findThreadById(long id)
|
|
||||||
{
|
|
||||||
for (Thread thread : Thread.getAllStackTraces().keySet())
|
|
||||||
{
|
|
||||||
if (thread.getId() == id)
|
|
||||||
{
|
|
||||||
return thread;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
throw new IllegalStateException("Deadlocked Thread not found!");
|
|
||||||
}
|
|
||||||
}
|
|
@ -21,6 +21,7 @@ import java.awt.Toolkit;
|
|||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.io.FileInputStream;
|
import java.io.FileInputStream;
|
||||||
import java.io.InputStream;
|
import java.io.InputStream;
|
||||||
|
import java.time.Duration;
|
||||||
import java.util.Calendar;
|
import java.util.Calendar;
|
||||||
import java.util.logging.LogManager;
|
import java.util.logging.LogManager;
|
||||||
import java.util.logging.Logger;
|
import java.util.logging.Logger;
|
||||||
@ -30,7 +31,7 @@ import org.l2jmobius.commons.concurrent.ThreadPool;
|
|||||||
import org.l2jmobius.commons.database.DatabaseFactory;
|
import org.l2jmobius.commons.database.DatabaseFactory;
|
||||||
import org.l2jmobius.commons.enums.ServerMode;
|
import org.l2jmobius.commons.enums.ServerMode;
|
||||||
import org.l2jmobius.commons.util.Chronos;
|
import org.l2jmobius.commons.util.Chronos;
|
||||||
import org.l2jmobius.commons.util.DeadlockDetector;
|
import org.l2jmobius.commons.util.DeadLockDetector;
|
||||||
import org.l2jmobius.commons.util.PropertiesParser;
|
import org.l2jmobius.commons.util.PropertiesParser;
|
||||||
import org.l2jmobius.commons.util.Util;
|
import org.l2jmobius.commons.util.Util;
|
||||||
import org.l2jmobius.gameserver.cache.CrestCache;
|
import org.l2jmobius.gameserver.cache.CrestCache;
|
||||||
@ -139,7 +140,6 @@ public class GameServer
|
|||||||
|
|
||||||
private static TelnetStatusThread _statusServer;
|
private static TelnetStatusThread _statusServer;
|
||||||
private static GameServer INSTANCE;
|
private static GameServer INSTANCE;
|
||||||
|
|
||||||
public static final Calendar dateTimeServerStarted = Calendar.getInstance();
|
public static final Calendar dateTimeServerStarted = Calendar.getInstance();
|
||||||
|
|
||||||
public long getUsedMemoryMB()
|
public long getUsedMemoryMB()
|
||||||
@ -179,10 +179,6 @@ public class GameServer
|
|||||||
|
|
||||||
Util.printSection("ThreadPool");
|
Util.printSection("ThreadPool");
|
||||||
ThreadPool.init();
|
ThreadPool.init();
|
||||||
if (Config.DEADLOCKCHECK_INTIAL_TIME > 0)
|
|
||||||
{
|
|
||||||
ThreadPool.scheduleAtFixedRate(DeadlockDetector.getInstance(), Config.DEADLOCKCHECK_INTIAL_TIME, Config.DEADLOCKCHECK_DELAY_TIME);
|
|
||||||
}
|
|
||||||
|
|
||||||
Util.printSection("IdManager");
|
Util.printSection("IdManager");
|
||||||
IdManager.getInstance();
|
IdManager.getInstance();
|
||||||
@ -491,6 +487,19 @@ public class GameServer
|
|||||||
{
|
{
|
||||||
PrecautionaryRestartManager.getInstance();
|
PrecautionaryRestartManager.getInstance();
|
||||||
}
|
}
|
||||||
|
if (Config.DEADLOCK_DETECTOR)
|
||||||
|
{
|
||||||
|
final DeadLockDetector deadDetectThread = new DeadLockDetector(Duration.ofSeconds(Config.DEADLOCK_CHECK_INTERVAL), () ->
|
||||||
|
{
|
||||||
|
if (Config.RESTART_ON_DEADLOCK)
|
||||||
|
{
|
||||||
|
AnnouncementsTable.getInstance().announceToAll("Server has stability issues - restarting now.");
|
||||||
|
Shutdown.getInstance().startShutdown(null, 60, true);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
deadDetectThread.setDaemon(true);
|
||||||
|
deadDetectThread.start();
|
||||||
|
}
|
||||||
|
|
||||||
Util.printSection("Status");
|
Util.printSection("Status");
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user