Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,16 @@
import me.lucko.spark.common.command.sender.CommandSender;
import me.lucko.spark.common.monitor.cpu.CpuMonitor;
import me.lucko.spark.common.monitor.memory.GarbageCollectorStatistics;
import me.lucko.spark.common.monitor.memory.MemoryAllocationInfo;
import me.lucko.spark.common.monitor.memory.MemoryMonitor;
import me.lucko.spark.common.monitor.net.NetworkMonitor;
import me.lucko.spark.common.monitor.ping.PingStatistics;
import me.lucko.spark.common.monitor.ping.PlayerPingProvider;
import me.lucko.spark.common.monitor.tick.SparkTickStatistics;
import me.lucko.spark.common.monitor.tick.TickStatistics;
import me.lucko.spark.common.platform.PlatformInfo;
import me.lucko.spark.common.platform.PlatformStatisticsProvider;
import me.lucko.spark.common.platform.WorldMetricsCollector;
import me.lucko.spark.common.sampler.BackgroundSamplerManager;
import me.lucko.spark.common.sampler.SamplerContainer;
import me.lucko.spark.common.sampler.source.ClassSourceLookup;
Expand Down Expand Up @@ -83,6 +86,7 @@ public class SparkPlatform {
private final TickStatistics tickStatistics;
private final PingStatistics pingStatistics;
private final PlatformStatisticsProvider statisticsProvider;
private final WorldMetricsCollector worldMetricsCollector;
private final CommandManager commandManager;
private final AtomicBoolean enabled = new AtomicBoolean(false);
private Map<String, GarbageCollectorStatistics> startupGcStatistics = ImmutableMap.of();
Expand Down Expand Up @@ -128,6 +132,7 @@ public SparkPlatform(SparkPlugin plugin) {
this.pingStatistics = pingProvider != null ? new PingStatistics(pingProvider) : null;

this.statisticsProvider = new PlatformStatisticsProvider(this);
this.worldMetricsCollector = new WorldMetricsCollector(this);

this.commandManager = new CommandManager(this, this.configuration);
}
Expand All @@ -148,7 +153,12 @@ public void enable() {
if (this.pingStatistics != null) {
this.pingStatistics.start();
}

this.worldMetricsCollector.start();

CpuMonitor.ensureMonitoring();
MemoryMonitor.ensureMonitoring();
MemoryAllocationInfo.ensureMonitoring();
NetworkMonitor.ensureMonitoring();

// poll startup GC statistics after plugins & the world have loaded
Expand Down Expand Up @@ -176,6 +186,7 @@ public void disable() {
if (this.pingStatistics != null) {
this.pingStatistics.close();
}
this.worldMetricsCollector.close();

this.samplerContainer.close();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,15 @@

package me.lucko.spark.common.command.modules;

import me.lucko.bytesocks.client.BytesocksClient;
import me.lucko.spark.common.SparkPlatform;
import me.lucko.spark.common.activitylog.Activity;
import me.lucko.spark.common.command.Arguments;
import me.lucko.spark.common.command.Command;
import me.lucko.spark.common.command.CommandModule;
import me.lucko.spark.common.command.CommandResponseHandler;
import me.lucko.spark.common.command.sender.CommandSender;
import me.lucko.spark.common.command.tabcomplete.CompletionSupplier;
import me.lucko.spark.common.command.tabcomplete.TabCompleter;
import me.lucko.spark.common.monitor.cpu.CpuMonitor;
import me.lucko.spark.common.monitor.disk.DiskUsage;
Expand All @@ -42,6 +44,7 @@
import me.lucko.spark.common.util.MediaTypes;
import me.lucko.spark.common.util.RollingAverage;
import me.lucko.spark.common.util.StatisticFormatter;
import me.lucko.spark.common.ws.HealthReportViewerSocket;
import me.lucko.spark.proto.SparkProtos;
import net.kyori.adventure.text.Component;
import net.kyori.adventure.text.event.ClickEvent;
Expand All @@ -51,6 +54,9 @@
import java.lang.management.MemoryPoolMXBean;
import java.lang.management.MemoryType;
import java.lang.management.MemoryUsage;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
Expand All @@ -74,6 +80,31 @@ public class HealthModule implements CommandModule {

@Override
public void registerCommands(Consumer<Command> consumer) {
consumer.accept(Command.builder()
.aliases("health", "healthreport")
.allowSubCommand(true)
.argumentUsage("dashboard", "", null)
.argumentUsage("upload", "", null)
.argumentUsage("show", "memory", null)
.argumentUsage("show", "network", null)
.executor(HealthModule::healthReport)
.tabCompleter((platform, sender, arguments) -> {
List<String> opts = Collections.emptyList();
if (!arguments.isEmpty()) {
String subCommand = arguments.get(0);
if (subCommand.equals("show")) {
opts = new ArrayList<>(Arrays.asList("--memory", "--network"));
opts.removeAll(arguments);
}
}
return TabCompleter.create()
.at(0, CompletionSupplier.startsWith(Arrays.asList("dashboard", "upload", "show")))
.from(1, CompletionSupplier.startsWith(opts))
.complete(arguments);
})
.build()
);

consumer.accept(Command.builder()
.aliases("tps", "cpu")
.executor(HealthModule::tps)
Expand All @@ -88,16 +119,6 @@ public void registerCommands(Consumer<Command> consumer) {
.tabCompleter((platform, sender, arguments) -> TabCompleter.completeForOpts(arguments, "--player"))
.build()
);

consumer.accept(Command.builder()
.aliases("healthreport", "health", "ht")
.argumentUsage("upload", null)
.argumentUsage("memory", null)
.argumentUsage("network", null)
.executor(HealthModule::healthReport)
.tabCompleter((platform, sender, arguments) -> TabCompleter.completeForOpts(arguments, "--upload", "--memory", "--network"))
.build()
);
}

private static void tps(SparkPlatform platform, CommandSender sender, CommandResponseHandler resp, Arguments arguments) {
Expand Down Expand Up @@ -192,12 +213,17 @@ private static void ping(SparkPlatform platform, CommandSender sender, CommandRe
}

private static void healthReport(SparkPlatform platform, CommandSender sender, CommandResponseHandler resp, Arguments arguments) {
resp.replyPrefixed(text("Generating server health report..."));
String subCommand = arguments.subCommand() == null ? "" : arguments.subCommand();

if (arguments.boolFlag("upload")) {
uploadHealthReport(platform, sender, resp, arguments);
return;
if (subCommand.equals("show")) {
healthReportShow(platform, sender, resp, arguments);
} else {
healthReportUpload(platform, sender, resp, !subCommand.equals("upload"));
}
}

private static void healthReportShow(SparkPlatform platform, CommandSender sender, CommandResponseHandler resp, Arguments arguments) {
resp.replyPrefixed(text("Generating server health report..."));

List<Component> report = new LinkedList<>();
report.add(empty());
Expand All @@ -223,7 +249,9 @@ private static void healthReport(SparkPlatform platform, CommandSender sender, C
resp.reply(report);
}

private static void uploadHealthReport(SparkPlatform platform, CommandSender sender, CommandResponseHandler resp, Arguments arguments) {
private static void healthReportUpload(SparkPlatform platform, CommandSender sender, CommandResponseHandler resp, boolean dashboard) {
resp.replyPrefixed(text("Generating server health report..."));

SparkProtos.HealthMetadata.Builder metadata = SparkProtos.HealthMetadata.newBuilder();
SparkMetadata.gather(platform, sender.toData(), platform.getStartupGcStatistics()).writeTo(metadata);

Expand All @@ -235,18 +263,33 @@ private static void uploadHealthReport(SparkPlatform platform, CommandSender sen
data.putAllTimeWindowStatistics(activeSampler.exportWindowStatistics());
}

if (dashboard) {
BytesocksClient bytesocksClient = platform.getBytesocksClient();
if (bytesocksClient == null) {
resp.replyPrefixed(text("The live viewer is not supported.", RED));
return;
}

try {
HealthReportViewerSocket socket = new HealthReportViewerSocket(platform, bytesocksClient);
data.setChannelInfo(socket.getPayload());
} catch (Exception e) {
resp.replyPrefixed(text("An error occurred whilst opening the live viewer connection.", RED));
platform.getPlugin().log(Level.WARNING, "Error whilst opening live viewer connection", e);
}
}

try {
String key = platform.getBytebinClient().postContent(data.build(), MediaTypes.SPARK_HEALTH_MEDIA_TYPE).key();
String url = platform.getViewerUrl() + key;

resp.broadcastPrefixed(text("Health report:", GOLD));
resp.broadcastPrefixed(text("Health Report:", GOLD));
resp.broadcast(text()
.content(url)
.color(GRAY)
.clickEvent(ClickEvent.openUrl(url))
.build()
);

platform.getActivityLog().addToLog(Activity.urlActivity(resp.senderData(), System.currentTimeMillis(), "Health report", url));
} catch (Exception e) {
resp.broadcastPrefixed(text("An error occurred whilst uploading the data.", RED));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
import me.lucko.spark.common.util.FormatUtil;
import me.lucko.spark.common.util.MediaTypes;
import me.lucko.spark.common.util.TimeUtil;
import me.lucko.spark.common.ws.SamplerViewerSocket;
import me.lucko.spark.common.ws.ViewerSocket;
import me.lucko.spark.proto.SparkSamplerProtos;
import net.kyori.adventure.text.Component;
Expand Down Expand Up @@ -463,7 +464,7 @@ private void handleUpload(SparkPlatform platform, CommandResponseHandler resp, S

private void handleOpen(SparkPlatform platform, BytesocksClient bytesocksClient, CommandResponseHandler resp, Sampler sampler, Sampler.ExportProps exportProps) {
try {
ViewerSocket socket = new ViewerSocket(platform, bytesocksClient, exportProps);
SamplerViewerSocket socket = new SamplerViewerSocket(platform, bytesocksClient, exportProps);
sampler.attachSocket(socket);
exportProps.channelInfo(socket.getPayload());

Expand Down
107 changes: 107 additions & 0 deletions spark-common/src/main/java/me/lucko/spark/common/monitor/Metrics.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
/*
* This file is part of spark.
*
* Copyright (c) lucko (Luck) <luck@lucko.me>
* Copyright (c) contributors
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/

package me.lucko.spark.common.monitor;

import me.lucko.spark.common.sampler.window.WindowStatisticsCollector;
import me.lucko.spark.common.util.MetricSeries;
import me.lucko.spark.common.util.TimeUtil;
import me.lucko.spark.proto.SparkProtos;

import java.time.Duration;

/**
* A collection of metrics series used for monitoring the server.
*
* <p>The metric series have a fixed retention period and an interval between recordings.
* The retention period determines how long the recorded metrics are kept,
* while the interval determines how often new metrics are recorded.</p>
*
* <p>These metrics are recorded at a higher interval than those collected by {@link WindowStatisticsCollector}.</p>
*/
public enum Metrics {
;

/** The retention period of the metrics series. */
private static final Duration RETENTION = Duration.ofHours(1);

/** The interval between metric recordings. */
public static final int INTERVAL_MILLIS = (int) Duration.ofSeconds(10).toMillis();

/** The estimated capacity of the metrics series. */
private static final int INITIAL_CAPACITY = Math.toIntExact(RETENTION.toMillis() / INTERVAL_MILLIS) + 1;

/**
* The timestamp after which metrics recording should start.
*
* <p>A delay avoids recording incomplete values when the server first starts</p>
*/
private static final long START_RECORDING_MILLIS = TimeUtil.monotonicCurrentTimeMillis() + INTERVAL_MILLIS;

public static final MetricSeries.Doubles TPS = new MetricSeries.Doubles(RETENTION, INITIAL_CAPACITY);
public static final MetricSeries.Averages TICK_DURATION = new MetricSeries.Averages(RETENTION, INITIAL_CAPACITY);
public static final MetricSeries.Doubles CPU_USAGE_PROCESS = new MetricSeries.Doubles(RETENTION, INITIAL_CAPACITY);
public static final MetricSeries.Doubles CPU_USAGE_SYSTEM = new MetricSeries.Doubles(RETENTION, INITIAL_CAPACITY);
public static final MetricSeries.MemoryUsages MEMORY_USAGE_HEAP = new MetricSeries.MemoryUsages(RETENTION, INITIAL_CAPACITY);
public static final MetricSeries.MemoryUsages MEMORY_USAGE_NON_HEAP = new MetricSeries.MemoryUsages(RETENTION, INITIAL_CAPACITY);
public static final MetricSeries.Doubles MEMORY_ALLOCATION = new MetricSeries.Doubles(RETENTION, INITIAL_CAPACITY);
public static final MetricSeries.WorldInfo WORLD_INFO = new MetricSeries.WorldInfo(RETENTION, INITIAL_CAPACITY);
public static final MetricSeries.Averages PLAYER_PING = new MetricSeries.Averages(RETENTION, INITIAL_CAPACITY);

public static boolean shouldRecordTps() {
return shouldRecord(TPS, TimeUtil.monotonicCurrentTimeMillis());
}

public static boolean shouldRecordTickDuration() {
return shouldRecord(TICK_DURATION, TimeUtil.monotonicCurrentTimeMillis());
}

public static boolean shouldRecordCpuUsageProcess(long timeNow) {
return shouldRecord(CPU_USAGE_PROCESS, timeNow);
}

public static boolean shouldRecordCpuUsageSystem(long timeNow) {
return shouldRecord(CPU_USAGE_SYSTEM, timeNow);
}

private static boolean shouldRecord(MetricSeries<?> series, long timeNow) {
if (timeNow < START_RECORDING_MILLIS) {
return false;
}

long newestTimestamp = series.newestTimestamp();
return newestTimestamp == 0 || newestTimestamp < timeNow - INTERVAL_MILLIS;
}

public static SparkProtos.Metrics exportProto() {
SparkProtos.Metrics.Builder builder = SparkProtos.Metrics.newBuilder();
if (!TPS.isEmpty()) builder.setTps(TPS.toProto());
if (!TICK_DURATION.isEmpty()) builder.setTickDuration(TICK_DURATION.toProto());
if (!CPU_USAGE_PROCESS.isEmpty()) builder.setCpuUsageProcess(CPU_USAGE_PROCESS.toProto());
if (!CPU_USAGE_SYSTEM.isEmpty()) builder.setCpuUsageSystem(CPU_USAGE_SYSTEM.toProto());
if (!MEMORY_USAGE_HEAP.isEmpty()) builder.setMemoryUsageHeap(MEMORY_USAGE_HEAP.toProto());
if (!MEMORY_USAGE_NON_HEAP.isEmpty()) builder.setMemoryUsageNonHeap(MEMORY_USAGE_NON_HEAP.toProto());
if (!MEMORY_ALLOCATION.isEmpty()) builder.setMemoryAllocation(MEMORY_ALLOCATION.toProto());
if (!WORLD_INFO.isEmpty()) builder.setWorldInfo(WORLD_INFO.toProto());
if (!PLAYER_PING.isEmpty()) builder.setPlayerPing(PLAYER_PING.toProto());
return builder.build();
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,21 @@
import me.lucko.spark.common.util.SparkThreadFactory;

import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.ThreadLocalRandom;

import static java.util.concurrent.TimeUnit.MILLISECONDS;

public enum MonitoringExecutor {
;

/** The executor used to monitor & calculate rolling averages. */
public static final ScheduledExecutorService INSTANCE = new SparkScheduledThreadPoolExecutor(1, new SparkThreadFactory("spark-monitoring", true));
public static final ScheduledExecutorService INSTANCE = new SparkScheduledThreadPoolExecutor(4, new SparkThreadFactory("spark-monitoring", true));

public static ScheduledFuture<?> scheduleAtFixedRateMillis(Runnable command, long periodMillis) {
// schedule the task with a random initial delay to avoid all fixed rate tasks running at the same time
long delay = ThreadLocalRandom.current().nextLong(Math.min(periodMillis, 10_000L));
return INSTANCE.scheduleAtFixedRate(command, delay, periodMillis, MILLISECONDS);
}

}
Loading
Loading