Newer
Older
/*
* This file is part of vospace-file-service
* Copyright (C) 2021 Istituto Nazionale di Astrofisica
* SPDX-License-Identifier: GPL-3.0-or-later
*/
package it.inaf.ia2.transfer.service;
Sonia Zorba
committed
import it.inaf.ia2.transfer.auth.TokenPrincipal;
import it.inaf.ia2.transfer.persistence.FileDAO;
Sonia Zorba
committed
import it.inaf.ia2.transfer.persistence.JobDAO;
import it.inaf.ia2.transfer.persistence.LocationDAO;
import it.inaf.ia2.transfer.persistence.model.FileInfo;
Sonia Zorba
committed
import it.inaf.ia2.transfer.persistence.model.JobException;
import it.inaf.ia2.transfer.persistence.model.JobException.Type;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
Sonia Zorba
committed
import java.io.OutputStream;
import java.nio.file.Files;
import java.security.Principal;
Sonia Zorba
committed
import java.util.Map;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import javax.annotation.PostConstruct;
import net.ivoa.xml.uws.v1.ExecutionPhase;
import org.kamranzafar.jtar.TarEntry;
import org.kamranzafar.jtar.TarOutputStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
Sonia Zorba
committed
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.stereotype.Service;
import org.springframework.util.FileSystemUtils;
Sonia Zorba
committed
import org.springframework.web.client.RestTemplate;
@Service
public class ArchiveService {
private static final Logger LOG = LoggerFactory.getLogger(ArchiveService.class);
@Autowired
private FileDAO fileDAO;
Sonia Zorba
committed
@Autowired
private LocationDAO locationDAO;
@Autowired
private JobDAO jobDAO;
@Autowired
private AuthorizationService authorizationService;
Sonia Zorba
committed
@Autowired
private RestTemplate restTemplate;
@Value("${upload_location_id}")
private int uploadLocationId;
@Value("${generated.dir}")
private String generatedDirString;
private File generatedDir;
Sonia Zorba
committed
@PostConstruct
public void init() {
this.generatedDir = new File(generatedDirString);
if (!this.generatedDir.exists()) {
if (!this.generatedDir.mkdirs()) {
throw new IllegalStateException("Unable to create directory " + this.generatedDir.getAbsolutePath());
}
}
}
Sonia Zorba
committed
public <O extends OutputStream, E> void createArchive(ArchiveJob job) {
jobDAO.updateJobPhase(ExecutionPhase.EXECUTING, job.getJobId());
LOG.trace("Started archive job " + job.getJobId());
try {
// TODO: check total size limit
Sonia Zorba
committed
File archiveFile = getArchiveFile(job);
String commonParent = getCommonParent(job.getVosPaths());
// support directory used to generate folder inside tar files (path is redefined each time by TarEntry class)
File supportDir = Files.createTempDirectory("dir").toFile();
Sonia Zorba
committed
// it will be initialized only when necessary
Map<Integer, String> portalLocationUrls = null;
try ( ArchiveHandler<O, E> handler = getArchiveHandler(archiveFile, job.getType())) {
for (FileInfo fileInfo : fileDAO.getArchiveFileInfos(job.getVosPaths())) {
String relPath = fileInfo.getVirtualPath().substring(commonParent.length());
if (fileInfo.isDirectory()) {
Sonia Zorba
committed
handler.putNextEntry(supportDir, relPath);
Sonia Zorba
committed
if (fileInfo.getLocationId() != null && fileInfo.getLocationId() != uploadLocationId) {
// remote file
if (portalLocationUrls == null) {
portalLocationUrls = locationDAO.getPortalLocationUrls();
}
String url = portalLocationUrls.get(fileInfo.getLocationId());
downloadFileIntoArchive(fileInfo, relPath, job.getPrincipal(), handler, url);
} else {
// local file or virtual directory
writeFileIntoArchive(fileInfo, relPath, job.getPrincipal(), handler);
}
}
} finally {
FileSystemUtils.deleteRecursively(supportDir);
Sonia Zorba
committed
jobDAO.updateJobPhase(ExecutionPhase.COMPLETED, job.getJobId());
} catch (Throwable t) {
Sonia Zorba
committed
JobException jobException;
if (t instanceof JobException) {
jobException = (JobException) t;
} else {
LOG.error("Unexpected error happened creating archive", t);
jobException = new JobException(Type.FATAL).setErrorMessage("Internal Fault")
.setErrorDetail("InternalFault: Unexpected error happened creating archive");
}
jobDAO.setJobError(job.getJobId(), jobException);
}
}
private File getArchiveFile(ArchiveJob job) throws IOException {
File parentDir = getArchiveParentDir(job.getPrincipal());
if (!parentDir.exists()) {
if (!parentDir.mkdirs()) {
LOG.error("Unable to create directory " + parentDir.getAbsolutePath());
throw new JobException(Type.FATAL).setErrorMessage("Internal Fault")
.setErrorDetail("InternalFault: Unable to create temporary directory for job");
}
Sonia Zorba
committed
File archiveFile = parentDir.toPath().resolve(job.getJobId() + "." + job.getType().getExtension()).toFile();
if (!archiveFile.createNewFile()) {
LOG.error("Unable to create file " + archiveFile.getAbsolutePath());
throw new JobException(Type.FATAL).setErrorMessage("Internal Fault")
.setErrorDetail("InternalFault: Unable to create archive file");
}
return archiveFile;
}
public File getArchiveParentDir(Principal principal) {
return generatedDir.toPath().resolve(principal.getName()).toFile();
}
private String getCommonParent(List<String> vosPaths) {
String commonParent = null;
for (String vosPath : vosPaths) {
if (commonParent == null) {
commonParent = vosPath;
} else {
StringBuilder newCommonParent = new StringBuilder();
boolean same = true;
for (int i = 0; same && i < Math.min(commonParent.length(), vosPath.length()); i++) {
if (commonParent.charAt(i) == vosPath.charAt(i)) {
newCommonParent.append(commonParent.charAt(i));
} else {
same = false;
}
}
commonParent = newCommonParent.toString();
}
}
return commonParent;
}
Sonia Zorba
committed
private static abstract class ArchiveHandler<O extends OutputStream, E> implements AutoCloseable {
Sonia Zorba
committed
private final O os;
Sonia Zorba
committed
ArchiveHandler(O os) {
this.os = os;
}
public abstract E getEntry(File file, String path);
public abstract void putNextEntry(E entry) throws IOException;
public void putNextEntry(File file, String path) throws IOException {
putNextEntry(getEntry(file, path));
}
public final O getOutputStream() {
return os;
}
Sonia Zorba
committed
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
@Override
public void close() throws Exception {
os.close();
}
}
private class TarArchiveHandler extends ArchiveHandler<TarOutputStream, TarEntry> {
TarArchiveHandler(File archiveFile) throws IOException {
super(new TarOutputStream(new BufferedOutputStream(new FileOutputStream(archiveFile))));
}
@Override
public TarEntry getEntry(File file, String path) {
return new TarEntry(file, path);
}
@Override
public void putNextEntry(TarEntry entry) throws IOException {
getOutputStream().putNextEntry(entry);
}
}
private class ZipArchiveHandler extends ArchiveHandler<ZipOutputStream, ZipEntry> {
ZipArchiveHandler(File archiveFile) throws IOException {
super(new ZipOutputStream(new BufferedOutputStream(new FileOutputStream(archiveFile))));
}
@Override
public ZipEntry getEntry(File file, String path) {
if (file.isDirectory()) {
// ZipEntry assumes that paths ending with / are folders
path += "/";
Sonia Zorba
committed
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
return new ZipEntry(path);
}
@Override
public void putNextEntry(ZipEntry entry) throws IOException {
getOutputStream().putNextEntry(entry);
}
}
private ArchiveHandler getArchiveHandler(File archiveFile, ArchiveJob.Type type) throws IOException {
switch (type) {
case TAR:
return new TarArchiveHandler(archiveFile);
case ZIP:
return new ZipArchiveHandler(archiveFile);
default:
throw new UnsupportedOperationException("Type " + type + " not supported yet");
}
}
private <O extends OutputStream, E> void downloadFileIntoArchive(FileInfo fileInfo, String relPath, TokenPrincipal tokenPrincipal, ArchiveHandler<O, E> handler, String baseUrl) {
if (baseUrl == null) {
LOG.error("Location URL not found for location " + fileInfo.getLocationId());
throw new JobException(Type.FATAL).setErrorMessage("Internal Fault")
.setErrorDetail("InternalFault: Unable to retrieve location of file " + fileInfo.getVirtualPath());
}
String url = baseUrl + "/" + fileInfo.getVirtualName();
LOG.trace("Downloading file from " + url);
restTemplate.execute(url, HttpMethod.GET, req -> {
HttpHeaders headers = req.getHeaders();
if (tokenPrincipal.getToken() != null) {
headers.setBearerAuth(tokenPrincipal.getToken());
}
}, res -> {
File tmpFile = Files.createTempFile("download", null).toFile();
try ( FileOutputStream os = new FileOutputStream(tmpFile)) {
res.getBody().transferTo(os);
handler.putNextEntry(tmpFile, relPath);
try ( FileInputStream is = new FileInputStream(tmpFile)) {
is.transferTo(handler.getOutputStream());
}
} finally {
tmpFile.delete();
}
return null;
}, new Object[]{});
}
private <O extends OutputStream, E> void writeFileIntoArchive(FileInfo fileInfo, String relPath, TokenPrincipal tokenPrincipal, ArchiveHandler<O, E> handler) throws IOException {
if (!authorizationService.isDownloadable(fileInfo, tokenPrincipal)) {
throw new JobException(Type.FATAL).setErrorMessage("Permission Denied")
.setErrorDetail("PermissionDenied: " + fileInfo.getVirtualPath());
}
File file = new File(fileInfo.getOsPath());
LOG.trace("Adding file " + file.getAbsolutePath() + " to tar archive");
try ( InputStream is = new FileInputStream(file)) {
handler.putNextEntry(file, relPath);
is.transferTo(handler.getOutputStream());