|
|
@@ -0,0 +1,370 @@
|
|
|
+package com.vas.kittyfallwsvas.wsfw.resultrequest;
|
|
|
+
|
|
|
+import com.google.gson.Gson;
|
|
|
+import java.io.BufferedReader;
|
|
|
+import java.io.BufferedWriter;
|
|
|
+import java.io.File;
|
|
|
+import java.io.FileReader;
|
|
|
+import java.io.IOException;
|
|
|
+import java.nio.ByteBuffer;
|
|
|
+import java.nio.channels.FileChannel;
|
|
|
+import java.nio.charset.StandardCharsets;
|
|
|
+import java.nio.file.AtomicMoveNotSupportedException;
|
|
|
+import java.nio.file.DirectoryStream;
|
|
|
+import java.nio.file.Files;
|
|
|
+import java.nio.file.Path;
|
|
|
+import java.nio.file.StandardCopyOption;
|
|
|
+import java.nio.file.StandardOpenOption;
|
|
|
+import java.util.ArrayList;
|
|
|
+import java.util.List;
|
|
|
+import java.util.Properties;
|
|
|
+import java.util.concurrent.ArrayBlockingQueue;
|
|
|
+import java.util.concurrent.CompletableFuture;
|
|
|
+import java.util.concurrent.Executors;
|
|
|
+import java.util.concurrent.ScheduledExecutorService;
|
|
|
+import java.util.concurrent.ThreadFactory;
|
|
|
+import java.util.concurrent.TimeUnit;
|
|
|
+import java.util.concurrent.TimeoutException;
|
|
|
+import java.util.concurrent.atomic.AtomicLong;
|
|
|
+import org.apache.log4j.Logger;
|
|
|
+import utils.Config;
|
|
|
+
|
|
|
+/**
|
|
|
+ * Persists accepted requests before replying and drains them to the database
|
|
|
+ * with a single, rate-limited worker.
|
|
|
+ */
|
|
|
+public class ResultRequestSpoolService {
|
|
|
+
|
|
|
+ public interface Processor {
|
|
|
+ void process(ResultRequestEvent event, String sourceFile, long sourceLine) throws Exception;
|
|
|
+ }
|
|
|
+
|
|
|
+ private static class Pending {
|
|
|
+ private final ResultRequestEvent event;
|
|
|
+ private final CompletableFuture<Void> durable = new CompletableFuture<Void>();
|
|
|
+
|
|
|
+ Pending(ResultRequestEvent event) {
|
|
|
+ this.event = event;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ private final Logger logger;
|
|
|
+ private final Processor processor;
|
|
|
+ private final Gson gson = new Gson();
|
|
|
+ private final boolean enabled;
|
|
|
+ private final Path root;
|
|
|
+ private final Path activeDir;
|
|
|
+ private final Path readyDir;
|
|
|
+ private final Path processingDir;
|
|
|
+ private final Path doneDir;
|
|
|
+ private final Path deadDir;
|
|
|
+ private final Path activeFile;
|
|
|
+ private final ArrayBlockingQueue<Pending> queue;
|
|
|
+ private final int batchSize;
|
|
|
+ private final long enqueueTimeoutMs;
|
|
|
+ private final long writeTimeoutMs;
|
|
|
+ private final int maxRetry;
|
|
|
+ private final long rateDelayMs;
|
|
|
+ private final long intervalSeconds;
|
|
|
+ private final Object activeLock = new Object();
|
|
|
+ private final AtomicLong sequence = new AtomicLong();
|
|
|
+ private volatile boolean running = true;
|
|
|
+ private Thread writerThread;
|
|
|
+ private ScheduledExecutorService scheduler;
|
|
|
+
|
|
|
+ public static ResultRequestSpoolService create(Logger logger, Processor processor) throws Exception {
|
|
|
+ Properties properties = new Properties();
|
|
|
+ File configFile = new File(Config.configDir, "app.conf");
|
|
|
+ FileReader reader = new FileReader(configFile);
|
|
|
+ try {
|
|
|
+ properties.load(reader);
|
|
|
+ } finally {
|
|
|
+ reader.close();
|
|
|
+ }
|
|
|
+ return new ResultRequestSpoolService(logger, processor, properties);
|
|
|
+ }
|
|
|
+
|
|
|
+ ResultRequestSpoolService(Logger logger, Processor processor, Properties properties) throws Exception {
|
|
|
+ this.logger = logger;
|
|
|
+ this.processor = processor;
|
|
|
+ this.enabled = "SPOOL".equalsIgnoreCase(get(properties, "RESULT_REQUEST_MODE", "DIRECT"));
|
|
|
+ this.root = new File(get(properties, "RESULT_REQUEST_SPOOL_DIR", "../data/result-request-spool")).toPath();
|
|
|
+ this.activeDir = root.resolve("active");
|
|
|
+ this.readyDir = root.resolve("ready");
|
|
|
+ this.processingDir = root.resolve("processing");
|
|
|
+ this.doneDir = root.resolve("done");
|
|
|
+ this.deadDir = root.resolve("dead-letter");
|
|
|
+ this.activeFile = activeDir.resolve("result-request.open");
|
|
|
+ this.batchSize = positiveInt(properties, "RESULT_REQUEST_WRITE_BATCH_SIZE", 100);
|
|
|
+ this.enqueueTimeoutMs = positiveLong(properties, "RESULT_REQUEST_ENQUEUE_TIMEOUT_MS", 2000L);
|
|
|
+ this.writeTimeoutMs = positiveLong(properties, "RESULT_REQUEST_WRITE_TIMEOUT_MS", 10000L);
|
|
|
+ this.maxRetry = positiveInt(properties, "RESULT_REQUEST_MAX_RETRY", 20);
|
|
|
+ this.intervalSeconds = positiveLong(properties, "RESULT_REQUEST_INTERVAL_SECONDS", 300L);
|
|
|
+ int rate = positiveInt(properties, "RESULT_REQUEST_RATE_LIMIT_PER_SECOND", 10);
|
|
|
+ this.rateDelayMs = Math.max(1L, 1000L / rate);
|
|
|
+ this.queue = new ArrayBlockingQueue<Pending>(positiveInt(properties, "RESULT_REQUEST_QUEUE_CAPACITY", 10000));
|
|
|
+
|
|
|
+ if (enabled) {
|
|
|
+ Files.createDirectories(activeDir);
|
|
|
+ Files.createDirectories(readyDir);
|
|
|
+ Files.createDirectories(processingDir);
|
|
|
+ Files.createDirectories(doneDir);
|
|
|
+ Files.createDirectories(deadDir);
|
|
|
+ recoverFiles();
|
|
|
+ startWriter();
|
|
|
+ startProcessor();
|
|
|
+ logger.info("resultRequest SPOOL enabled. directory=" + root.toAbsolutePath()
|
|
|
+ + ", intervalSeconds=" + intervalSeconds + ", dbRate=" + rate + "/s");
|
|
|
+ } else {
|
|
|
+ logger.info("resultRequest mode is DIRECT (spool worker is not started)");
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ public boolean isEnabled() {
|
|
|
+ return enabled;
|
|
|
+ }
|
|
|
+
|
|
|
+ public void enqueue(ResultRequestEvent event) throws Exception {
|
|
|
+ if (!enabled) {
|
|
|
+ throw new IllegalStateException("resultRequest spool is disabled");
|
|
|
+ }
|
|
|
+ Pending pending = new Pending(event);
|
|
|
+ if (!queue.offer(pending, enqueueTimeoutMs, TimeUnit.MILLISECONDS)) {
|
|
|
+ throw new IOException("resultRequest spool queue is full");
|
|
|
+ }
|
|
|
+ try {
|
|
|
+ pending.durable.get(writeTimeoutMs, TimeUnit.MILLISECONDS);
|
|
|
+ } catch (TimeoutException ex) {
|
|
|
+ throw new IOException("Timed out waiting for durable spool write", ex);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ private void startWriter() {
|
|
|
+ writerThread = new Thread(new Runnable() {
|
|
|
+ @Override
|
|
|
+ public void run() {
|
|
|
+ writerLoop();
|
|
|
+ }
|
|
|
+ }, "result-request-spool-writer");
|
|
|
+ writerThread.setDaemon(true);
|
|
|
+ writerThread.start();
|
|
|
+ }
|
|
|
+
|
|
|
+ private void writerLoop() {
|
|
|
+ while (running) {
|
|
|
+ List<Pending> batch = new ArrayList<Pending>(batchSize);
|
|
|
+ try {
|
|
|
+ Pending first = queue.take();
|
|
|
+ batch.add(first);
|
|
|
+ queue.drainTo(batch, batchSize - 1);
|
|
|
+ writeBatch(batch);
|
|
|
+ for (Pending pending : batch) {
|
|
|
+ pending.durable.complete(null);
|
|
|
+ }
|
|
|
+ } catch (InterruptedException ex) {
|
|
|
+ Thread.currentThread().interrupt();
|
|
|
+ return;
|
|
|
+ } catch (Exception ex) {
|
|
|
+ logger.error("Cannot write resultRequest spool batch", ex);
|
|
|
+ for (Pending pending : batch) {
|
|
|
+ pending.durable.completeExceptionally(ex);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ private void writeBatch(List<Pending> batch) throws IOException {
|
|
|
+ StringBuilder content = new StringBuilder();
|
|
|
+ for (Pending pending : batch) {
|
|
|
+ content.append(gson.toJson(pending.event)).append('\n');
|
|
|
+ }
|
|
|
+ byte[] bytes = content.toString().getBytes(StandardCharsets.UTF_8);
|
|
|
+ synchronized (activeLock) {
|
|
|
+ FileChannel channel = FileChannel.open(activeFile, StandardOpenOption.CREATE,
|
|
|
+ StandardOpenOption.WRITE, StandardOpenOption.APPEND);
|
|
|
+ try {
|
|
|
+ ByteBuffer buffer = ByteBuffer.wrap(bytes);
|
|
|
+ while (buffer.hasRemaining()) {
|
|
|
+ channel.write(buffer);
|
|
|
+ }
|
|
|
+ channel.force(true);
|
|
|
+ } finally {
|
|
|
+ channel.close();
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ private void startProcessor() {
|
|
|
+ scheduler = Executors.newSingleThreadScheduledExecutor(new ThreadFactory() {
|
|
|
+ @Override
|
|
|
+ public Thread newThread(Runnable task) {
|
|
|
+ Thread thread = new Thread(task, "result-request-db-worker");
|
|
|
+ thread.setDaemon(true);
|
|
|
+ return thread;
|
|
|
+ }
|
|
|
+ });
|
|
|
+ scheduler.scheduleWithFixedDelay(new Runnable() {
|
|
|
+ @Override
|
|
|
+ public void run() {
|
|
|
+ try {
|
|
|
+ drainOnce();
|
|
|
+ } catch (Exception ex) {
|
|
|
+ logger.error("resultRequest spool drain error", ex);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }, intervalSeconds, intervalSeconds, TimeUnit.SECONDS);
|
|
|
+ }
|
|
|
+
|
|
|
+ private void drainOnce() throws Exception {
|
|
|
+ recoverProcessingFiles();
|
|
|
+ rotateActiveFile();
|
|
|
+ List<Path> files = listFiles(readyDir, "*.ready");
|
|
|
+ for (Path ready : files) {
|
|
|
+ Path claimed = processingDir.resolve(ready.getFileName().toString().replace(".ready", ".processing"));
|
|
|
+ move(ready, claimed);
|
|
|
+ processFile(claimed);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ private void rotateActiveFile() throws IOException {
|
|
|
+ synchronized (activeLock) {
|
|
|
+ if (!Files.exists(activeFile) || Files.size(activeFile) == 0) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ Path ready = readyDir.resolve(fileName("batch", ".ready"));
|
|
|
+ move(activeFile, ready);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ private void processFile(Path file) throws IOException {
|
|
|
+ BufferedReader reader = Files.newBufferedReader(file, StandardCharsets.UTF_8);
|
|
|
+ long lineNumber = 0;
|
|
|
+ try {
|
|
|
+ String line;
|
|
|
+ while ((line = reader.readLine()) != null) {
|
|
|
+ lineNumber++;
|
|
|
+ if (line.trim().length() == 0) {
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ ResultRequestEvent event;
|
|
|
+ try {
|
|
|
+ event = gson.fromJson(line, ResultRequestEvent.class);
|
|
|
+ if (event == null || event.getEventId() == null) {
|
|
|
+ throw new IllegalArgumentException("Missing eventId");
|
|
|
+ }
|
|
|
+ } catch (Exception ex) {
|
|
|
+ logger.error("Invalid resultRequest spool JSON " + file + ":" + lineNumber, ex);
|
|
|
+ writeDeadLine(line, file.getFileName().toString(), lineNumber);
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ try {
|
|
|
+ processor.process(event, file.getFileName().toString(), lineNumber);
|
|
|
+ } catch (Exception ex) {
|
|
|
+ logger.error("resultRequest DB processing failed eventId=" + event.getEventId(), ex);
|
|
|
+ event.incrementRetryCount();
|
|
|
+ if (event.getRetryCount() > maxRetry) {
|
|
|
+ writeEvent(deadDir, "failed", ".dead", event);
|
|
|
+ } else {
|
|
|
+ writeEvent(readyDir, "retry", ".ready", event);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ try {
|
|
|
+ Thread.sleep(rateDelayMs);
|
|
|
+ } catch (InterruptedException ex) {
|
|
|
+ Thread.currentThread().interrupt();
|
|
|
+ throw new IOException("Spool processor interrupted", ex);
|
|
|
+ }
|
|
|
+ }
|
|
|
+ } finally {
|
|
|
+ reader.close();
|
|
|
+ }
|
|
|
+ move(file, doneDir.resolve(file.getFileName().toString().replace(".processing", ".done")));
|
|
|
+ }
|
|
|
+
|
|
|
+ private void recoverFiles() throws IOException {
|
|
|
+ if (Files.exists(activeFile) && Files.size(activeFile) > 0) {
|
|
|
+ move(activeFile, readyDir.resolve(fileName("recovered", ".ready")));
|
|
|
+ }
|
|
|
+ recoverProcessingFiles();
|
|
|
+ }
|
|
|
+
|
|
|
+ private void recoverProcessingFiles() throws IOException {
|
|
|
+ for (Path processing : listFiles(processingDir, "*.processing")) {
|
|
|
+ move(processing, readyDir.resolve(fileName("recovered-processing", ".ready")));
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ private void writeEvent(Path directory, String prefix, String extension, ResultRequestEvent event) throws IOException {
|
|
|
+ writeAtomic(directory.resolve(fileName(prefix, extension)), gson.toJson(event) + "\n");
|
|
|
+ }
|
|
|
+
|
|
|
+ private void writeDeadLine(String line, String source, long lineNumber) throws IOException {
|
|
|
+ String value = "{\"source\":\"" + source.replace("\"", "") + "\",\"line\":"
|
|
|
+ + lineNumber + ",\"raw\":" + gson.toJson(line) + "}\n";
|
|
|
+ writeAtomic(deadDir.resolve(fileName("invalid", ".dead")), value);
|
|
|
+ }
|
|
|
+
|
|
|
+ private void writeAtomic(Path target, String content) throws IOException {
|
|
|
+ Path temp = target.resolveSibling(target.getFileName().toString() + ".tmp");
|
|
|
+ FileChannel channel = FileChannel.open(temp, StandardOpenOption.CREATE_NEW, StandardOpenOption.WRITE);
|
|
|
+ try {
|
|
|
+ ByteBuffer buffer = ByteBuffer.wrap(content.getBytes(StandardCharsets.UTF_8));
|
|
|
+ while (buffer.hasRemaining()) {
|
|
|
+ channel.write(buffer);
|
|
|
+ }
|
|
|
+ channel.force(true);
|
|
|
+ } finally {
|
|
|
+ channel.close();
|
|
|
+ }
|
|
|
+ move(temp, target);
|
|
|
+ }
|
|
|
+
|
|
|
+ private List<Path> listFiles(Path directory, String glob) throws IOException {
|
|
|
+ List<Path> result = new ArrayList<Path>();
|
|
|
+ DirectoryStream<Path> stream = Files.newDirectoryStream(directory, glob);
|
|
|
+ try {
|
|
|
+ for (Path file : stream) {
|
|
|
+ result.add(file);
|
|
|
+ }
|
|
|
+ } finally {
|
|
|
+ stream.close();
|
|
|
+ }
|
|
|
+ java.util.Collections.sort(result);
|
|
|
+ return result;
|
|
|
+ }
|
|
|
+
|
|
|
+ private void move(Path source, Path target) throws IOException {
|
|
|
+ try {
|
|
|
+ Files.move(source, target, StandardCopyOption.ATOMIC_MOVE);
|
|
|
+ } catch (AtomicMoveNotSupportedException ex) {
|
|
|
+ Files.move(source, target);
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ private String fileName(String prefix, String extension) {
|
|
|
+ return prefix + "-" + System.currentTimeMillis() + "-" + sequence.incrementAndGet() + extension;
|
|
|
+ }
|
|
|
+
|
|
|
+ private static String get(Properties properties, String name, String defaultValue) {
|
|
|
+ String value = properties.getProperty(name);
|
|
|
+ return value == null || value.trim().length() == 0 ? defaultValue : value.trim();
|
|
|
+ }
|
|
|
+
|
|
|
+ private static int positiveInt(Properties properties, String name, int defaultValue) {
|
|
|
+ try {
|
|
|
+ int value = Integer.parseInt(get(properties, name, String.valueOf(defaultValue)));
|
|
|
+ return value > 0 ? value : defaultValue;
|
|
|
+ } catch (Exception ex) {
|
|
|
+ return defaultValue;
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ private static long positiveLong(Properties properties, String name, long defaultValue) {
|
|
|
+ try {
|
|
|
+ long value = Long.parseLong(get(properties, name, String.valueOf(defaultValue)));
|
|
|
+ return value > 0 ? value : defaultValue;
|
|
|
+ } catch (Exception ex) {
|
|
|
+ return defaultValue;
|
|
|
+ }
|
|
|
+ }
|
|
|
+}
|
student