Newer
Older
/*
* This file is part of vospace-file-service
* Copyright (C) 2021 Istituto Nazionale di Astrofisica
* SPDX-License-Identifier: GPL-3.0-or-later
*/
package it.inaf.ia2.transfer.service;
Sonia Zorba
committed
import it.inaf.ia2.transfer.auth.TokenPrincipal;
import it.inaf.ia2.transfer.persistence.FileDAO;
Sonia Zorba
committed
import it.inaf.ia2.transfer.persistence.JobDAO;
import it.inaf.ia2.transfer.persistence.LocationDAO;
import it.inaf.ia2.transfer.persistence.model.FileInfo;
import it.inaf.oats.vospace.exception.InternalFaultException;
import it.inaf.oats.vospace.exception.PermissionDeniedException;
import it.inaf.oats.vospace.exception.QuotaExceededException;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
Sonia Zorba
committed
import java.io.OutputStream;
import java.io.UncheckedIOException;
import java.nio.file.Files;
import java.security.Principal;
Sonia Zorba
committed
import java.util.Map;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import javax.annotation.PostConstruct;
import net.ivoa.xml.uws.v1.ExecutionPhase;
import org.kamranzafar.jtar.TarEntry;
import org.kamranzafar.jtar.TarOutputStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
Sonia Zorba
committed
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.stereotype.Service;
import org.springframework.util.FileSystemUtils;
import org.springframework.util.unit.DataSize;
Sonia Zorba
committed
import org.springframework.web.client.RestTemplate;
@Service
public class ArchiveService {
private static final Logger LOG = LoggerFactory.getLogger(ArchiveService.class);
@Autowired
private FileDAO fileDAO;
Sonia Zorba
committed
@Autowired
private LocationDAO locationDAO;
@Autowired
private JobDAO jobDAO;
@Autowired
private AuthorizationService authorizationService;
Sonia Zorba
committed
@Autowired
private RestTemplate restTemplate;
@Value("${upload_location_id}")
private int uploadLocationId;
// Directory containing temporary files generated by jobs.
Sonia Zorba
committed
@Value("${generated.dir}")
private String generatedDirString;
private File generatedDir;
// Maximum size of the working directory for each registered user
@Value("${generated.dir.max-size}")
private DataSize generatedDirMaxSize;
Sonia Zorba
committed
@PostConstruct
public void init() {
this.generatedDir = new File(generatedDirString);
if (!this.generatedDir.exists()) {
if (!this.generatedDir.mkdirs()) {
throw new IllegalStateException("Unable to create directory " + this.generatedDir.getAbsolutePath());
}
}
}
Sonia Zorba
committed
public <O extends OutputStream, E> void createArchive(ArchiveJob job) {
jobDAO.updateJobPhase(ExecutionPhase.EXECUTING, job.getJobId());
LOG.trace("Started archive job " + job.getJobId());
try {
// TODO: check total size limit
Sonia Zorba
committed
File archiveFile = getArchiveFile(job);
String commonParent = getCommonParent(job.getVosPaths());
// support directory used to generate folder inside tar files (path is redefined each time by TarEntry class)
File supportDir = Files.createTempDirectory("dir").toFile();
Sonia Zorba
committed
// it will be initialized only when necessary
Map<Integer, String> portalLocationUrls = null;
try ( ArchiveHandler<O, E> handler = getArchiveHandler(archiveFile, job.getType())) {
for (FileInfo fileInfo : fileDAO.getArchiveFileInfos(job.getVosPaths())) {
String relPath = fileInfo.getVirtualPath().substring(commonParent.length());
if (fileInfo.isDirectory()) {
Sonia Zorba
committed
handler.putNextEntry(supportDir, relPath);
Sonia Zorba
committed
if (fileInfo.getLocationId() != null && fileInfo.getLocationId() != uploadLocationId) {
// remote file
if (portalLocationUrls == null) {
portalLocationUrls = locationDAO.getPortalLocationUrls();
}
String url = portalLocationUrls.get(fileInfo.getLocationId());
downloadFileIntoArchive(fileInfo, relPath, job.getPrincipal(), handler, url);
} else {
// local file or virtual directory
writeFileIntoArchive(fileInfo, relPath, job.getPrincipal(), handler);
}
}
} finally {
FileSystemUtils.deleteRecursively(supportDir);
Sonia Zorba
committed
} catch (IOException ex) {
throw new UncheckedIOException(ex);
Sonia Zorba
committed
}
}
private File getArchiveFile(ArchiveJob job) throws IOException {
File parentDir = getArchiveParentDir(job.getPrincipal());
if (!parentDir.exists()) {
if (!parentDir.mkdirs()) {
LOG.error("Unable to create directory " + parentDir.getAbsolutePath());
throw new InternalFaultException("Unable to create temporary directory for job");
Sonia Zorba
committed
}
Sonia Zorba
committed
checkQuotaLimit(parentDir);
Sonia Zorba
committed
File archiveFile = parentDir.toPath().resolve(job.getJobId() + "." + job.getType().getExtension()).toFile();
Sonia Zorba
committed
if (!archiveFile.createNewFile()) {
LOG.error("Unable to create file " + archiveFile.getAbsolutePath());
throw new InternalFaultException("Unable to create archive file");
Sonia Zorba
committed
}
return archiveFile;
}
/**
* If used working space exceeds quota limit throws an
* InsufficientStorageException.
*/
private void checkQuotaLimit(File parentDir) throws IOException {
long usedSpace = Files.walk(parentDir.toPath()).mapToLong(p -> p.toFile().length()).sum();
if (usedSpace > generatedDirMaxSize.toBytes()) {
throw new QuotaExceededException("Archive size limit exceeded.");
public File getArchiveParentDir(Principal principal) {
return generatedDir.toPath().resolve(principal.getName()).toFile();
}
private String getCommonParent(List<String> vosPaths) {
String commonParent = null;
for (String vosPath : vosPaths) {
if (commonParent == null) {
commonParent = vosPath;
} else {
StringBuilder newCommonParent = new StringBuilder();
boolean same = true;
int lastSlashPos = vosPath.lastIndexOf("/");
for (int i = 0; same && i < Math.min(commonParent.length(), vosPath.length()) && i <= lastSlashPos; i++) {
if (commonParent.charAt(i) == vosPath.charAt(i)) {
newCommonParent.append(commonParent.charAt(i));
} else {
same = false;
}
}
commonParent = newCommonParent.toString();
}
}
return commonParent;
}
private abstract class ArchiveHandler<O extends OutputStream, E> implements AutoCloseable {
Sonia Zorba
committed
private final O os;
private final File parentDir;
ArchiveHandler(O os, File parentDir) {
Sonia Zorba
committed
this.os = os;
this.parentDir = parentDir;
Sonia Zorba
committed
}
public abstract E getEntry(File file, String path);
protected abstract void putNextEntry(E entry) throws IOException;
Sonia Zorba
committed
public final void putNextEntry(File file, String path) throws IOException {
Sonia Zorba
committed
putNextEntry(getEntry(file, path));
checkQuotaLimit(parentDir);
Sonia Zorba
committed
}
public final O getOutputStream() {
return os;
}
Sonia Zorba
committed
@Override
public void close() throws IOException {
Sonia Zorba
committed
os.close();
}
}
private class TarArchiveHandler extends ArchiveHandler<TarOutputStream, TarEntry> {
TarArchiveHandler(File archiveFile) throws IOException {
super(new TarOutputStream(new BufferedOutputStream(new FileOutputStream(archiveFile))), archiveFile.getParentFile());
Sonia Zorba
committed
}
@Override
public TarEntry getEntry(File file, String path) {
return new TarEntry(file, path);
}
@Override
public void putNextEntry(TarEntry entry) throws IOException {
getOutputStream().putNextEntry(entry);
}
}
private class ZipArchiveHandler extends ArchiveHandler<ZipOutputStream, ZipEntry> {
ZipArchiveHandler(File archiveFile) throws IOException {
super(new ZipOutputStream(new BufferedOutputStream(new FileOutputStream(archiveFile))), archiveFile.getParentFile());
Sonia Zorba
committed
}
@Override
public ZipEntry getEntry(File file, String path) {
if (file.isDirectory()) {
// ZipEntry assumes that paths ending with / are folders
path += "/";
Sonia Zorba
committed
return new ZipEntry(path);
}
@Override
public void putNextEntry(ZipEntry entry) throws IOException {
getOutputStream().putNextEntry(entry);
}
}
private ArchiveHandler getArchiveHandler(File archiveFile, ArchiveJob.Type type) throws IOException {
switch (type) {
case TAR:
return new TarArchiveHandler(archiveFile);
case ZIP:
return new ZipArchiveHandler(archiveFile);
default:
throw new UnsupportedOperationException("Type " + type + " not supported yet");
}
}
private <O extends OutputStream, E> void downloadFileIntoArchive(FileInfo fileInfo, String relPath, TokenPrincipal tokenPrincipal, ArchiveHandler<O, E> handler, String baseUrl) {
if (baseUrl == null) {
LOG.error("Location URL not found for location " + fileInfo.getLocationId());
throw new InternalFaultException("Unable to retrieve location of file " + fileInfo.getVirtualPath());
Sonia Zorba
committed
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
}
String url = baseUrl + "/" + fileInfo.getVirtualName();
LOG.trace("Downloading file from " + url);
restTemplate.execute(url, HttpMethod.GET, req -> {
HttpHeaders headers = req.getHeaders();
if (tokenPrincipal.getToken() != null) {
headers.setBearerAuth(tokenPrincipal.getToken());
}
}, res -> {
File tmpFile = Files.createTempFile("download", null).toFile();
try ( FileOutputStream os = new FileOutputStream(tmpFile)) {
res.getBody().transferTo(os);
handler.putNextEntry(tmpFile, relPath);
try ( FileInputStream is = new FileInputStream(tmpFile)) {
is.transferTo(handler.getOutputStream());
}
} finally {
tmpFile.delete();
}
return null;
}, new Object[]{});
}
private <O extends OutputStream, E> void writeFileIntoArchive(FileInfo fileInfo, String relPath, TokenPrincipal tokenPrincipal, ArchiveHandler<O, E> handler) throws IOException {
if (!authorizationService.isDownloadable(fileInfo, tokenPrincipal)) {
throw PermissionDeniedException.forPath(fileInfo.getVirtualPath());
Sonia Zorba
committed
}
File file = new File(fileInfo.getOsPath());
LOG.trace("Adding file " + file.getAbsolutePath() + " to tar archive");
try ( InputStream is = new FileInputStream(file)) {
handler.putNextEntry(file, relPath);
is.transferTo(handler.getOutputStream());