Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,11 @@
*/
package io.pixelsdb.pixels.cli.executor;

import io.grpc.StatusRuntimeException;
import io.pixelsdb.pixels.cli.load.*;
import io.pixelsdb.pixels.common.exception.MetadataException;
import io.pixelsdb.pixels.common.exception.RetinaException;
import io.pixelsdb.pixels.common.exception.TransException;
import io.pixelsdb.pixels.common.metadata.MetadataService;
import io.pixelsdb.pixels.common.metadata.domain.File;
import io.pixelsdb.pixels.common.metadata.domain.Path;
Expand All @@ -44,7 +46,10 @@
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicReference;

/**
* @author hank
Expand Down Expand Up @@ -77,8 +82,43 @@ public void execute(Namespace ns, String command) throws Exception
MetadataService metadataService = MetadataService.Instance();
ConcurrentLinkedQueue<LoadedInfo> loadedInfos = new ConcurrentLinkedQueue<>();
long startTime = System.currentTimeMillis();
ScheduledExecutorService renewal = Executors.newSingleThreadScheduledExecutor(task -> {
Thread thread = new Thread(task, "load-lease-" + context.getTransId());
thread.setDaemon(true);
return thread;
});
Object leaseLock = new Object();
AtomicReference<TransException> leaseFailure = new AtomicReference<>();
try
{
long intervalMs = Math.max(1, context.getLease().getPeriodMs() / 10);
renewal.scheduleWithFixedDelay(() -> {
synchronized (leaseLock)
{
if (renewal.isShutdown())
{
return;
}
try
{
if (!transService.extendTransLease(context))
{
throw new TransException("transaction lease has expired");
}
} catch (StatusRuntimeException e)
{
// Transient RPC failure: the lease is still valid, so retry on the next tick.
System.err.println("failed to renew LOAD transaction " + context.getTransId() +
", will retry: " + e.getStatus());
} catch (Exception e)
{
leaseFailure.set(new TransException(
"failed to renew LOAD transaction " + context.getTransId(), e));
renewal.shutdown();
}
}
}, intervalMs, intervalMs, TimeUnit.MILLISECONDS);

Storage storage = StorageFactory.Instance().getStorage(origin);
Parameters parameters = new Parameters(schemaName, tableName, rowNum, regex,
encodingLevel, nullsPadding, metadataService, context.getTransId(), context.getTimestamp());
Expand All @@ -91,11 +131,15 @@ public void execute(Namespace ns, String command) throws Exception
inputFiles.add(storage.ensureSchemePrefix(filePath));
}

startConsumers(threadNum, inputFiles, parameters, loadedInfos);
startConsumers(threadNum, inputFiles, parameters, loadedInfos, leaseFailure);

int retinaServerPort = Integer.parseInt(ConfigFactory.Instance().getProperty("retina.server.port"));
for(LoadedInfo loadedInfo : loadedInfos)
{
if (leaseFailure.get() != null)
{
throw leaseFailure.get();
}
File file = loadedInfo.loadedFile;
Path path = loadedInfo.loadedPath;
if (!metadataService.updateFile(file))
Expand Down Expand Up @@ -127,7 +171,33 @@ public void execute(Namespace ns, String command) throws Exception
}
}

transService.commitTrans(context.getTransId(), false);
// Wait for any in-flight renewal and stop further renewals before committing.
synchronized (leaseLock)
{
renewal.shutdown();
if (leaseFailure.get() != null)
{
throw leaseFailure.get();
}
// Transient RPC failures are retried until the lease itself expires, same as the renewal task.
while (true)
{
try
{
if (!transService.extendTransLease(context))
{
throw new TransException("LOAD transaction lease has expired");
}
break;
} catch (StatusRuntimeException e)
{
System.err.println("failed to renew LOAD transaction " + context.getTransId() +
" before commit, will retry: " + e.getStatus());
Thread.sleep(intervalMs);
}
}
transService.commitTrans(context.getTransId(), false);
}
System.out.println(command + " is successful");
} catch (Exception failure)
{
Expand Down Expand Up @@ -164,6 +234,7 @@ public void execute(Namespace ns, String command) throws Exception
throw failure;
} finally
{
renewal.shutdownNow();
long endTime = System.currentTimeMillis();
System.out.println("Text files in '" + origin + "' are loaded by " + threadNum +
" threads in " + (endTime - startTime) / 1000.0 + "s.");
Expand All @@ -176,10 +247,12 @@ public void execute(Namespace ns, String command) throws Exception
* @param inputFiles the queue of the paths of input files
* @param parameters the parameters for data loading, e.g., the schema name and table name
* @param loadedInfos the information of the loaded pixels files
* @param leaseFailure a failure reported by the transaction renewal task
* @throws Exception if parameters cannot be initialized or a consumer fails
*/
private void startConsumers(int concurrency, BlockingQueue<String> inputFiles, Parameters parameters,
ConcurrentLinkedQueue<LoadedInfo> loadedInfos) throws Exception
ConcurrentLinkedQueue<LoadedInfo> loadedInfos,
AtomicReference<TransException> leaseFailure) throws Exception
{
if (!parameters.initExtra())
{
Expand All @@ -205,7 +278,21 @@ private void startConsumers(int concurrency, BlockingQueue<String> inputFiles, P

for (Future<?> future : futures)
{
future.get();
while (true)
{
if (leaseFailure.get() != null)
{
throw leaseFailure.get();
}
try
{
future.get(1, TimeUnit.SECONDS);
break;
} catch (TimeoutException ignored)
{
// Check renewal failures even when a consumer is blocked on storage.
}
}
}
} catch (ExecutionException e)
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
import io.pixelsdb.pixels.daemon.NodeProto;

import java.io.IOException;
import java.io.InterruptedIOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
Expand Down Expand Up @@ -137,6 +138,10 @@ public void run()
}

// Flush any remaining data from writers
if (Thread.currentThread().isInterrupted())
{
throw new InterruptedIOException("Data loading interrupted");
}
flushRemainingData();

} catch (InterruptedException e)
Expand All @@ -151,7 +156,17 @@ public void run()
throw new RuntimeException(this.getClass().getSimpleName() + " failed", e);
} finally
{
cleanupTemporaryFiles();
boolean interrupted = Thread.interrupted();
try
{
cleanupTemporaryFiles();
} finally
{
if (interrupted)
{
Thread.currentThread().interrupt();
}
}
System.out.println(Thread.currentThread().getName() + ":" + count);
System.out.println("Exit " + this.getClass().getSimpleName() + ", thread: " + Thread.currentThread().getName() +
", time: " + DateUtil.formatTime(new Date()));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@
import java.io.BufferedReader;
import java.io.DataInputStream;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.List;
Expand Down Expand Up @@ -91,6 +92,10 @@ protected void processSourceFile(String originalFilePath) throws IOException, Me

while ((line = reader.readLine()) != null)
{
if (Thread.currentThread().isInterrupted())
{
throw new InterruptedIOException("Data loading interrupted");
}
lineNumber++;

// limit=-1 keeps trailing empty fields; String.split(regex) drops them by default.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@

import java.io.BufferedReader;
import java.io.IOException;
import java.io.InterruptedIOException;
import java.io.InputStreamReader;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.ConcurrentLinkedQueue;
Expand Down Expand Up @@ -73,6 +74,10 @@ protected void processSourceFile(String originalFilePath) throws IOException, Me

while ((line = reader.readLine()) != null)
{
if (Thread.currentThread().isInterrupted())
{
throw new InterruptedIOException("Data loading interrupted");
}
lineNumber++;

// 1. Check if a new PixelsFile needs to be initialized
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -295,7 +295,9 @@ public boolean extendTransLease(TransContext transContext) throws TransException
}
TransProto.ExtendTransLeaseRequest request = TransProto.ExtendTransLeaseRequest.newBuilder()
.setTransId(transContext.getTransId()).build();
TransProto.ExtendTransLeaseResponse response = this.stub.extendTransLease(request);
TransProto.ExtendTransLeaseResponse response = this.stub
.withDeadlineAfter(Math.max(1, lease.getPeriodMs() / 10), TimeUnit.MILLISECONDS)
.extendTransLease(request);
if (response.getErrorCode() != ErrorCode.SUCCESS)
{
throw new TransException("transaction " + transContext.getTransId() +
Expand Down
Loading