From 085abe3f749baa63b60544d33d4c485ea3f47832 Mon Sep 17 00:00:00 2001 From: Dongyang Geng Date: Thu, 12 Nov 2026 11:45:10 +0800 Subject: [PATCH 1/2] fix(cli): renew transaction leases during LOAD --- .../pixels/cli/executor/LoadExecutor.java | 82 ++++++++++++++++++- .../cli/load/AbstractPixelsConsumer.java | 17 +++- .../cli/load/IndexedPixelsConsumer.java | 5 ++ .../pixels/cli/load/SimplePixelsConsumer.java | 5 ++ .../common/transaction/TransService.java | 4 +- 5 files changed, 107 insertions(+), 6 deletions(-) diff --git a/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/executor/LoadExecutor.java b/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/executor/LoadExecutor.java index 380a09672..abcb6cfe1 100644 --- a/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/executor/LoadExecutor.java +++ b/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/executor/LoadExecutor.java @@ -19,9 +19,11 @@ */ package io.pixelsdb.pixels.cli.executor; +import io.grpc.StatusRuntimeException; import io.pixelsdb.pixels.cli.load.*; import io.pixelsdb.pixels.common.exception.MetadataException; import io.pixelsdb.pixels.common.exception.RetinaException; +import io.pixelsdb.pixels.common.exception.TransException; import io.pixelsdb.pixels.common.metadata.MetadataService; import io.pixelsdb.pixels.common.metadata.domain.File; import io.pixelsdb.pixels.common.metadata.domain.Path; @@ -44,7 +46,10 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicReference; /** * @author hank @@ -77,8 +82,43 @@ public void execute(Namespace ns, String command) throws Exception MetadataService metadataService = MetadataService.Instance(); ConcurrentLinkedQueue loadedInfos = new ConcurrentLinkedQueue<>(); long startTime = System.currentTimeMillis(); + ScheduledExecutorService renewal = Executors.newSingleThreadScheduledExecutor(task -> { + Thread thread = new Thread(task, "load-lease-" + context.getTransId()); + thread.setDaemon(true); + return thread; + }); + Object leaseLock = new Object(); + AtomicReference leaseFailure = new AtomicReference<>(); try { + long intervalMs = Math.max(1, context.getLease().getPeriodMs() / 10); + renewal.scheduleWithFixedDelay(() -> { + synchronized (leaseLock) + { + if (renewal.isShutdown()) + { + return; + } + try + { + if (!transService.extendTransLease(context)) + { + throw new TransException("transaction lease has expired"); + } + } catch (StatusRuntimeException e) + { + // Transient RPC failure: the lease is still valid, so retry on the next tick. + System.err.println("failed to renew LOAD transaction " + context.getTransId() + + ", will retry: " + e.getStatus()); + } catch (Exception e) + { + leaseFailure.set(new TransException( + "failed to renew LOAD transaction " + context.getTransId(), e)); + renewal.shutdown(); + } + } + }, intervalMs, intervalMs, TimeUnit.MILLISECONDS); + Storage storage = StorageFactory.Instance().getStorage(origin); Parameters parameters = new Parameters(schemaName, tableName, rowNum, regex, encodingLevel, nullsPadding, metadataService, context.getTransId(), context.getTimestamp()); @@ -91,11 +131,15 @@ public void execute(Namespace ns, String command) throws Exception inputFiles.add(storage.ensureSchemePrefix(filePath)); } - startConsumers(threadNum, inputFiles, parameters, loadedInfos); + startConsumers(threadNum, inputFiles, parameters, loadedInfos, leaseFailure); int retinaServerPort = Integer.parseInt(ConfigFactory.Instance().getProperty("retina.server.port")); for(LoadedInfo loadedInfo : loadedInfos) { + if (leaseFailure.get() != null) + { + throw leaseFailure.get(); + } File file = loadedInfo.loadedFile; Path path = loadedInfo.loadedPath; if (!metadataService.updateFile(file)) @@ -127,7 +171,20 @@ public void execute(Namespace ns, String command) throws Exception } } - transService.commitTrans(context.getTransId(), false); + // Wait for any in-flight renewal and stop further renewals before committing. + synchronized (leaseLock) + { + renewal.shutdown(); + if (leaseFailure.get() != null) + { + throw leaseFailure.get(); + } + if (!transService.extendTransLease(context)) + { + throw new TransException("LOAD transaction lease has expired"); + } + transService.commitTrans(context.getTransId(), false); + } System.out.println(command + " is successful"); } catch (Exception failure) { @@ -164,6 +221,7 @@ public void execute(Namespace ns, String command) throws Exception throw failure; } finally { + renewal.shutdownNow(); long endTime = System.currentTimeMillis(); System.out.println("Text files in '" + origin + "' are loaded by " + threadNum + " threads in " + (endTime - startTime) / 1000.0 + "s."); @@ -176,10 +234,12 @@ public void execute(Namespace ns, String command) throws Exception * @param inputFiles the queue of the paths of input files * @param parameters the parameters for data loading, e.g., the schema name and table name * @param loadedInfos the information of the loaded pixels files + * @param leaseFailure a failure reported by the transaction renewal task * @throws Exception if parameters cannot be initialized or a consumer fails */ private void startConsumers(int concurrency, BlockingQueue inputFiles, Parameters parameters, - ConcurrentLinkedQueue loadedInfos) throws Exception + ConcurrentLinkedQueue loadedInfos, + AtomicReference leaseFailure) throws Exception { if (!parameters.initExtra()) { @@ -205,7 +265,21 @@ private void startConsumers(int concurrency, BlockingQueue inputFiles, P for (Future future : futures) { - future.get(); + while (true) + { + if (leaseFailure.get() != null) + { + throw leaseFailure.get(); + } + try + { + future.get(1, TimeUnit.SECONDS); + break; + } catch (TimeoutException ignored) + { + // Check renewal failures even when a consumer is blocked on storage. + } + } } } catch (ExecutionException e) { diff --git a/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/load/AbstractPixelsConsumer.java b/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/load/AbstractPixelsConsumer.java index 3923e5190..52d832653 100644 --- a/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/load/AbstractPixelsConsumer.java +++ b/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/load/AbstractPixelsConsumer.java @@ -39,6 +39,7 @@ import io.pixelsdb.pixels.daemon.NodeProto; import java.io.IOException; +import java.io.InterruptedIOException; import java.util.ArrayList; import java.util.Collections; import java.util.Date; @@ -137,6 +138,10 @@ public void run() } // Flush any remaining data from writers + if (Thread.currentThread().isInterrupted()) + { + throw new InterruptedIOException("Data loading interrupted"); + } flushRemainingData(); } catch (InterruptedException e) @@ -151,7 +156,17 @@ public void run() throw new RuntimeException(this.getClass().getSimpleName() + " failed", e); } finally { - cleanupTemporaryFiles(); + boolean interrupted = Thread.interrupted(); + try + { + cleanupTemporaryFiles(); + } finally + { + if (interrupted) + { + Thread.currentThread().interrupt(); + } + } System.out.println(Thread.currentThread().getName() + ":" + count); System.out.println("Exit " + this.getClass().getSimpleName() + ", thread: " + Thread.currentThread().getName() + ", time: " + DateUtil.formatTime(new Date())); diff --git a/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/load/IndexedPixelsConsumer.java b/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/load/IndexedPixelsConsumer.java index fd9f9fee2..170fac47c 100644 --- a/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/load/IndexedPixelsConsumer.java +++ b/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/load/IndexedPixelsConsumer.java @@ -47,6 +47,7 @@ import java.io.BufferedReader; import java.io.DataInputStream; import java.io.IOException; + import java.io.InterruptedIOException; import java.io.InputStreamReader; import java.util.ArrayList; import java.util.List; @@ -91,6 +92,10 @@ protected void processSourceFile(String originalFilePath) throws IOException, Me while ((line = reader.readLine()) != null) { + if (Thread.currentThread().isInterrupted()) + { + throw new InterruptedIOException("Data loading interrupted"); + } lineNumber++; // limit=-1 keeps trailing empty fields; String.split(regex) drops them by default. diff --git a/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/load/SimplePixelsConsumer.java b/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/load/SimplePixelsConsumer.java index 444563128..785c2442e 100644 --- a/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/load/SimplePixelsConsumer.java +++ b/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/load/SimplePixelsConsumer.java @@ -35,6 +35,7 @@ import java.io.BufferedReader; import java.io.IOException; + import java.io.InterruptedIOException; import java.io.InputStreamReader; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentLinkedQueue; @@ -73,6 +74,10 @@ protected void processSourceFile(String originalFilePath) throws IOException, Me while ((line = reader.readLine()) != null) { + if (Thread.currentThread().isInterrupted()) + { + throw new InterruptedIOException("Data loading interrupted"); + } lineNumber++; // 1. Check if a new PixelsFile needs to be initialized diff --git a/pixels-common/src/main/java/io/pixelsdb/pixels/common/transaction/TransService.java b/pixels-common/src/main/java/io/pixelsdb/pixels/common/transaction/TransService.java index d0e5be784..a8846ea64 100644 --- a/pixels-common/src/main/java/io/pixelsdb/pixels/common/transaction/TransService.java +++ b/pixels-common/src/main/java/io/pixelsdb/pixels/common/transaction/TransService.java @@ -295,7 +295,9 @@ public boolean extendTransLease(TransContext transContext) throws TransException } TransProto.ExtendTransLeaseRequest request = TransProto.ExtendTransLeaseRequest.newBuilder() .setTransId(transContext.getTransId()).build(); - TransProto.ExtendTransLeaseResponse response = this.stub.extendTransLease(request); + TransProto.ExtendTransLeaseResponse response = this.stub + .withDeadlineAfter(Math.max(1, lease.getPeriodMs() / 10), TimeUnit.MILLISECONDS) + .extendTransLease(request); if (response.getErrorCode() != ErrorCode.SUCCESS) { throw new TransException("transaction " + transContext.getTransId() + From 32085129c3e80ba4ec0304411ad81d6fd5553857 Mon Sep 17 00:00:00 2001 From: Dongyang Geng Date: Fri, 13 Nov 2026 11:30:15 +0800 Subject: [PATCH 2/2] fix(cli): retry transient RPC failures in pre-commit lease renewal --- .../pixels/cli/executor/LoadExecutor.java | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/executor/LoadExecutor.java b/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/executor/LoadExecutor.java index abcb6cfe1..f29a40fc5 100644 --- a/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/executor/LoadExecutor.java +++ b/pixels-cli/src/main/java/io/pixelsdb/pixels/cli/executor/LoadExecutor.java @@ -179,9 +179,22 @@ public void execute(Namespace ns, String command) throws Exception { throw leaseFailure.get(); } - if (!transService.extendTransLease(context)) + // Transient RPC failures are retried until the lease itself expires, same as the renewal task. + while (true) { - throw new TransException("LOAD transaction lease has expired"); + try + { + if (!transService.extendTransLease(context)) + { + throw new TransException("LOAD transaction lease has expired"); + } + break; + } catch (StatusRuntimeException e) + { + System.err.println("failed to renew LOAD transaction " + context.getTransId() + + " before commit, will retry: " + e.getStatus()); + Thread.sleep(intervalMs); + } } transService.commitTrans(context.getTransId(), false); }