Skip to content
Commits on Source (9)
...@@ -15,8 +15,13 @@ import org.springframework.boot.autoconfigure.SpringBootApplication; ...@@ -15,8 +15,13 @@ import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.web.servlet.FilterRegistrationBean; import org.springframework.boot.web.servlet.FilterRegistrationBean;
import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Bean;
import org.springframework.web.client.RestTemplate; import org.springframework.web.client.RestTemplate;
import org.springframework.context.annotation.Import;
import it.inaf.oats.vospace.parent.persistence.LinkedServiceDAO;
import it.inaf.ia2.aa.ServiceLocator;
import it.inaf.ia2.aa.ServletRapClient;
@SpringBootApplication @SpringBootApplication
@Import(LinkedServiceDAO.class)
public class FileServiceApplication { public class FileServiceApplication {
@Value("${jwks_uri}") @Value("${jwks_uri}")
...@@ -39,6 +44,11 @@ public class FileServiceApplication { ...@@ -39,6 +44,11 @@ public class FileServiceApplication {
registration.addUrlPatterns("/*"); registration.addUrlPatterns("/*");
return registration; return registration;
} }
@Bean
public ServletRapClient servletRapClient() {
return (ServletRapClient) ServiceLocator.getInstance().getRapClient();
}
@Bean @Bean
public RestTemplate restTemplate() { public RestTemplate restTemplate() {
......
...@@ -12,6 +12,7 @@ import it.inaf.ia2.transfer.service.ArchiveService; ...@@ -12,6 +12,7 @@ import it.inaf.ia2.transfer.service.ArchiveService;
import it.inaf.oats.vospace.exception.PermissionDeniedException; import it.inaf.oats.vospace.exception.PermissionDeniedException;
import java.io.File; import java.io.File;
import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletableFuture;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpServletResponse;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpHeaders; import org.springframework.http.HttpHeaders;
...@@ -29,6 +30,9 @@ public class ArchiveFileController extends AuthenticatedFileController { ...@@ -29,6 +30,9 @@ public class ArchiveFileController extends AuthenticatedFileController {
@Autowired @Autowired
private ArchiveService archiveService; private ArchiveService archiveService;
@Autowired
private HttpServletRequest servletRequest;
@Autowired @Autowired
private HttpServletResponse response; private HttpServletResponse response;
...@@ -42,10 +46,10 @@ public class ArchiveFileController extends AuthenticatedFileController { ...@@ -42,10 +46,10 @@ public class ArchiveFileController extends AuthenticatedFileController {
job.setPrincipal(getPrincipal()); job.setPrincipal(getPrincipal());
job.setJobId(archiveRequest.getJobId()); job.setJobId(archiveRequest.getJobId());
job.setType(type); job.setType(type);
job.setVosPaths(archiveRequest.getPaths()); job.setEntryDescriptors(archiveRequest.getEntryDescriptors());
CompletableFuture.runAsync(() -> { CompletableFuture.runAsync(() -> {
handleFileJob(() -> archiveService.createArchive(job), job.getJobId()); handleFileJob(() -> archiveService.createArchive(job, servletRequest), job.getJobId());
}); });
HttpHeaders headers = new HttpHeaders(); HttpHeaders headers = new HttpHeaders();
......
...@@ -5,13 +5,14 @@ ...@@ -5,13 +5,14 @@
*/ */
package it.inaf.ia2.transfer.controller; package it.inaf.ia2.transfer.controller;
import it.inaf.oats.vospace.parent.exchange.ArchiveEntryDescriptor;
import java.util.List; import java.util.List;
public class ArchiveRequest { public class ArchiveRequest {
private String type; private String type;
private String jobId; private String jobId;
private List<String> paths; private List<ArchiveEntryDescriptor> entryDescriptors;
public String getType() { public String getType() {
return type; return type;
...@@ -29,11 +30,11 @@ public class ArchiveRequest { ...@@ -29,11 +30,11 @@ public class ArchiveRequest {
this.jobId = jobId; this.jobId = jobId;
} }
public List<String> getPaths() { public List<ArchiveEntryDescriptor> getEntryDescriptors() {
return paths; return entryDescriptors;
} }
public void setPaths(List<String> paths) { public void setEntryDescriptors(List<ArchiveEntryDescriptor> entryDescriptors) {
this.paths = paths; this.entryDescriptors = entryDescriptors;
} }
} }
...@@ -47,7 +47,7 @@ public class FileDAO { ...@@ -47,7 +47,7 @@ public class FileDAO {
+ "accept_views, provide_views, l.location_type, n.path <> n.relative_path AS virtual_parent,\n" + "accept_views, provide_views, l.location_type, n.path <> n.relative_path AS virtual_parent,\n"
+ "(SELECT user_name FROM users WHERE user_id = creator_id) AS username, n.job_id,\n" + "(SELECT user_name FROM users WHERE user_id = creator_id) AS username, n.job_id,\n"
+ "base_path, get_os_path(n.node_id) AS os_path, ? AS vos_path, false AS is_directory,\n" + "base_path, get_os_path(n.node_id) AS os_path, ? AS vos_path, false AS is_directory,\n"
+ "type = 'link' AS is_link,\n" + "n.type = 'link' AS is_link, n.target,\n"
+ "fs_path \n" + "fs_path \n"
+ "FROM node n\n" + "FROM node n\n"
+ "JOIN location l ON (n.location_id IS NOT NULL AND n.location_id = l.location_id) OR (n.location_id IS NULL AND l.location_id = ?)\n" + "JOIN location l ON (n.location_id IS NOT NULL AND n.location_id = l.location_id) OR (n.location_id IS NULL AND l.location_id = ?)\n"
...@@ -180,7 +180,7 @@ public class FileDAO { ...@@ -180,7 +180,7 @@ public class FileDAO {
+ "(SELECT user_name FROM users WHERE user_id = n.creator_id) AS username,\n" + "(SELECT user_name FROM users WHERE user_id = n.creator_id) AS username,\n"
+ "base_path, get_os_path(n.node_id) AS os_path, get_vos_path(n.node_id) AS vos_path,\n" + "base_path, get_os_path(n.node_id) AS os_path, get_vos_path(n.node_id) AS vos_path,\n"
+ "n.type = 'container' AS is_directory, n.name, n.location_id, n.job_id,\n" + "n.type = 'container' AS is_directory, n.name, n.location_id, n.job_id,\n"
+ "n.type = 'link' AS is_link, l.location_type\n" + "n.type = 'link' AS is_link, n.target, l.location_type\n"
+ "FROM node n\n" + "FROM node n\n"
+ "JOIN node p ON p.path @> n.path\n" + "JOIN node p ON p.path @> n.path\n"
+ "LEFT JOIN location l ON l.location_id = n.location_id\n" + "LEFT JOIN location l ON l.location_id = n.location_id\n"
...@@ -213,7 +213,7 @@ public class FileDAO { ...@@ -213,7 +213,7 @@ public class FileDAO {
+ "(SELECT user_name FROM users WHERE user_id = n.creator_id) AS username,\n" + "(SELECT user_name FROM users WHERE user_id = n.creator_id) AS username,\n"
+ "base_path, get_os_path(n.node_id) AS os_path, get_vos_path(n.node_id) AS vos_path,\n" + "base_path, get_os_path(n.node_id) AS os_path, get_vos_path(n.node_id) AS vos_path,\n"
+ "n.type = 'container' AS is_directory, n.name, n.location_id, n.job_id,\n" + "n.type = 'container' AS is_directory, n.name, n.location_id, n.job_id,\n"
+ "n.type = 'link' AS is_link, l.location_type\n" + "n.type = 'link' AS is_link, n.target, l.location_type\n"
+ "FROM node n\n" + "FROM node n\n"
+ "JOIN node p ON p.path @> n.path\n" + "JOIN node p ON p.path @> n.path\n"
+ "LEFT JOIN location l ON l.location_id = n.location_id\n" + "LEFT JOIN location l ON l.location_id = n.location_id\n"
...@@ -289,11 +289,16 @@ public class FileDAO { ...@@ -289,11 +289,16 @@ public class FileDAO {
long contentLength = rs.getLong("content_length"); long contentLength = rs.getLong("content_length");
if (!rs.wasNull()) { if (!rs.wasNull()) {
fi.setContentLength(contentLength); fi.setContentLength(contentLength);
} }
fi.setContentMd5(rs.getString("content_md5")); fi.setContentMd5(rs.getString("content_md5"));
fi.setContentType(rs.getString("content_type")); fi.setContentType(rs.getString("content_type"));
fi.setDirectory(rs.getBoolean("is_directory")); fi.setDirectory(rs.getBoolean("is_directory"));
fi.setLink(rs.getBoolean("is_link")); if(rs.getBoolean("is_link")){
fi.setLink(true);
fi.setTarget(rs.getString("target"));
} else {
fi.setLink(false);
}
fi.setJobId(rs.getString("job_id")); fi.setJobId(rs.getString("job_id"));
int locationId = rs.getInt("location_id"); int locationId = rs.getInt("location_id");
if (!rs.wasNull()) { if (!rs.wasNull()) {
......
...@@ -17,11 +17,12 @@ public class FileInfo { ...@@ -17,11 +17,12 @@ public class FileInfo {
private String fsPath; private String fsPath;
// actualBasePath differs from base path in db due to some location type // actualBasePath differs from base path in db due to some location type
// dependent manipulations (performed by FileDAO) // dependent manipulations (performed by FileDAO)
private String actualBasePath; private String actualBasePath;
private boolean isPublic; private boolean isPublic;
private boolean virtualParent; private boolean virtualParent;
private boolean directory; private boolean directory;
private boolean link; private boolean link;
private String target;
private List<String> groupRead; private List<String> groupRead;
private List<String> groupWrite; private List<String> groupWrite;
private String ownerId; private String ownerId;
...@@ -36,6 +37,14 @@ public class FileInfo { ...@@ -36,6 +37,14 @@ public class FileInfo {
private String locationType; private String locationType;
private String jobId; private String jobId;
public String getTarget() {
return target;
}
public void setTarget(String target) {
this.target = target;
}
public int getNodeId() { public int getNodeId() {
return nodeId; return nodeId;
} }
......
...@@ -6,6 +6,7 @@ ...@@ -6,6 +6,7 @@
package it.inaf.ia2.transfer.service; package it.inaf.ia2.transfer.service;
import it.inaf.ia2.transfer.auth.TokenPrincipal; import it.inaf.ia2.transfer.auth.TokenPrincipal;
import it.inaf.oats.vospace.parent.exchange.ArchiveEntryDescriptor;
import java.util.List; import java.util.List;
public class ArchiveJob { public class ArchiveJob {
...@@ -26,17 +27,17 @@ public class ArchiveJob { ...@@ -26,17 +27,17 @@ public class ArchiveJob {
} }
} }
private List<String> vosPaths; private List<ArchiveEntryDescriptor> entryDescriptors;
private TokenPrincipal tokenPrincipal; private TokenPrincipal tokenPrincipal;
private String jobId; private String jobId;
private Type type; private Type type;
public List<String> getVosPaths() { public List<ArchiveEntryDescriptor> getEntryDescriptors() {
return vosPaths; return entryDescriptors;
} }
public void setVosPaths(List<String> vosPaths) { public void setEntryDescriptors(List<ArchiveEntryDescriptor> entryDescriptors) {
this.vosPaths = vosPaths; this.entryDescriptors = entryDescriptors;
} }
public TokenPrincipal getPrincipal() { public TokenPrincipal getPrincipal() {
...@@ -62,4 +63,5 @@ public class ArchiveJob { ...@@ -62,4 +63,5 @@ public class ArchiveJob {
public void setType(Type type) { public void setType(Type type) {
this.type = type; this.type = type;
} }
} }
...@@ -5,6 +5,9 @@ ...@@ -5,6 +5,9 @@
*/ */
package it.inaf.ia2.transfer.service; package it.inaf.ia2.transfer.service;
import it.inaf.ia2.aa.ServletRapClient;
import it.inaf.ia2.aa.data.User;
import it.inaf.ia2.rap.client.call.TokenExchangeRequest;
import it.inaf.ia2.transfer.auth.TokenPrincipal; import it.inaf.ia2.transfer.auth.TokenPrincipal;
import it.inaf.ia2.transfer.persistence.FileDAO; import it.inaf.ia2.transfer.persistence.FileDAO;
import it.inaf.ia2.transfer.persistence.JobDAO; import it.inaf.ia2.transfer.persistence.JobDAO;
...@@ -13,6 +16,8 @@ import it.inaf.ia2.transfer.persistence.model.FileInfo; ...@@ -13,6 +16,8 @@ import it.inaf.ia2.transfer.persistence.model.FileInfo;
import it.inaf.oats.vospace.exception.InternalFaultException; import it.inaf.oats.vospace.exception.InternalFaultException;
import it.inaf.oats.vospace.exception.PermissionDeniedException; import it.inaf.oats.vospace.exception.PermissionDeniedException;
import it.inaf.oats.vospace.exception.QuotaExceededException; import it.inaf.oats.vospace.exception.QuotaExceededException;
import it.inaf.oats.vospace.parent.exchange.ArchiveEntryDescriptor;
import it.inaf.oats.vospace.parent.persistence.LinkedServiceDAO;
import java.io.BufferedOutputStream; import java.io.BufferedOutputStream;
import java.io.File; import java.io.File;
import java.io.FileInputStream; import java.io.FileInputStream;
...@@ -25,9 +30,11 @@ import java.nio.file.Files; ...@@ -25,9 +30,11 @@ import java.nio.file.Files;
import java.security.Principal; import java.security.Principal;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.stream.Collectors;
import java.util.zip.ZipEntry; import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream; import java.util.zip.ZipOutputStream;
import javax.annotation.PostConstruct; import javax.annotation.PostConstruct;
import javax.servlet.http.HttpServletRequest;
import net.ivoa.xml.uws.v1.ExecutionPhase; import net.ivoa.xml.uws.v1.ExecutionPhase;
import org.kamranzafar.jtar.TarEntry; import org.kamranzafar.jtar.TarEntry;
import org.kamranzafar.jtar.TarOutputStream; import org.kamranzafar.jtar.TarOutputStream;
...@@ -53,6 +60,9 @@ public class ArchiveService { ...@@ -53,6 +60,9 @@ public class ArchiveService {
@Autowired @Autowired
private LocationDAO locationDAO; private LocationDAO locationDAO;
@Autowired
private LinkedServiceDAO linkedServiceDAO;
@Autowired @Autowired
private JobDAO jobDAO; private JobDAO jobDAO;
...@@ -62,6 +72,9 @@ public class ArchiveService { ...@@ -62,6 +72,9 @@ public class ArchiveService {
@Autowired @Autowired
private RestTemplate restTemplate; private RestTemplate restTemplate;
@Autowired
private ServletRapClient rapClient;
@Value("${upload_location_id}") @Value("${upload_location_id}")
private int uploadLocationId; private int uploadLocationId;
...@@ -84,7 +97,7 @@ public class ArchiveService { ...@@ -84,7 +97,7 @@ public class ArchiveService {
} }
} }
public <O extends OutputStream, E> void createArchive(ArchiveJob job) { public <O extends OutputStream, E> void createArchive(ArchiveJob job, HttpServletRequest servletRequest) {
jobDAO.updateJobPhase(ExecutionPhase.EXECUTING, job.getJobId()); jobDAO.updateJobPhase(ExecutionPhase.EXECUTING, job.getJobId());
...@@ -94,42 +107,112 @@ public class ArchiveService { ...@@ -94,42 +107,112 @@ public class ArchiveService {
// TODO: check total size limit // TODO: check total size limit
File archiveFile = getArchiveFile(job); File archiveFile = getArchiveFile(job);
String commonParent = getCommonParent(job.getVosPaths()); List<ArchiveEntryDescriptor> entryDescriptors = job.getEntryDescriptors();
String commonParent = getCommonParent(entryDescriptors);
// support directory used to generate folder inside tar files (path is redefined each time by TarEntry class) // support directory used to generate folder inside tar files (path is redefined each time by TarEntry class)
File supportDir = Files.createTempDirectory("dir").toFile(); File supportDir = Files.createTempDirectory("dir").toFile();
// it will be initialized only when necessary
Map<Integer, String> portalLocationUrls = null;
try ( ArchiveHandler<O, E> handler = getArchiveHandler(archiveFile, job.getType())) { try ( ArchiveHandler<O, E> handler = getArchiveHandler(archiveFile, job.getType())) {
fillArchive(entryDescriptors, commonParent, supportDir,
job.getPrincipal(), servletRequest, handler);
} finally {
FileSystemUtils.deleteRecursively(supportDir);
}
for (FileInfo fileInfo : fileDAO.getArchiveFileInfos(job.getVosPaths())) { } catch (IOException ex) {
throw new UncheckedIOException(ex);
}
}
String relPath = fileInfo.getVirtualPath().substring(commonParent.length()); private <O extends OutputStream, E> void fillArchive(
List<ArchiveEntryDescriptor> entryDescriptors, String commonParent,
File supportDir, TokenPrincipal tokenPrincipal,
HttpServletRequest servletRequest, ArchiveHandler<O, E> handler) throws IOException {
if (fileInfo.isDirectory()) { // it will be initialized only when necessary
handler.putNextEntry(supportDir, relPath); Map<Integer, String> portalLocationUrls = null;
continue;
}
if (fileInfo.getLocationId() != null && "portal".equals(fileInfo.getLocationType())) { List<ArchiveEntryDescriptor> noTargetEntryDescriptors
// remote file = entryDescriptors.stream().filter(ed -> !ed.isPointingToAnotherNode())
if (portalLocationUrls == null) { .collect(Collectors.toList());
portalLocationUrls = locationDAO.getPortalLocationUrls();
} // Start with archive entry descriptors which don't point to another node
String url = portalLocationUrls.get(fileInfo.getLocationId()); List<String> vosPaths = noTargetEntryDescriptors.stream()
downloadFileIntoArchive(fileInfo, relPath, job.getPrincipal(), handler, url); .map(ed -> ed.getVosPath())
} else { .collect(Collectors.toList());
// local file or virtual directory
writeFileIntoArchive(fileInfo, relPath, job.getPrincipal(), handler); if (!vosPaths.isEmpty()) {
} for (FileInfo fileInfo : fileDAO.getArchiveFileInfos(vosPaths)) {
String relPath = fileInfo.getVirtualPath().substring(commonParent.length());
this.insertEntryIntoArchive(fileInfo, supportDir, relPath, tokenPrincipal, portalLocationUrls, servletRequest, handler);
}
}
List<ArchiveEntryDescriptor> pointingEntryDescriptors
= entryDescriptors.stream().filter(ed -> ed.isPointingToAnotherNode())
.collect(Collectors.toList());
// Now archive entry descriptors pointing to another node
List<String> targetNodesVosPaths = pointingEntryDescriptors.stream()
.map(ed -> ed.getTargetNodeVosPath())
.collect(Collectors.toList());
if (!targetNodesVosPaths.isEmpty()) {
for (FileInfo fileInfo : fileDAO.getArchiveFileInfos(targetNodesVosPaths)) {
// relPaths is calculated from base node
String targetNodeVosPath = fileInfo.getVirtualPath();
List<String> linkVosPaths = pointingEntryDescriptors.stream()
.filter(ed -> ed.getTargetNodeVosPath().equals(targetNodeVosPath))
.map(ed -> ed.getVosPath())
.collect(Collectors.toList());
for (String vosPath : linkVosPaths) {
String relPath = vosPath.substring(commonParent.length());
this.insertEntryIntoArchive(fileInfo, supportDir, relPath, tokenPrincipal, portalLocationUrls, servletRequest, handler);
} }
} finally {
FileSystemUtils.deleteRecursively(supportDir);
} }
}
}
} catch (IOException ex) { private <O extends OutputStream, E> void insertEntryIntoArchive(
throw new UncheckedIOException(ex); FileInfo fileInfo, File supportDir, String relPath,
TokenPrincipal tokenPrincipal, Map<Integer, String> portalLocationUrls,
HttpServletRequest servletRequest, ArchiveHandler<O, E> handler)
throws IOException {
if (fileInfo.isDirectory()) {
handler.putNextEntry(supportDir, relPath);
return;
}
// I retrieve only external links
// local links have been resolved before calling this endpoint
// TODO: we need to discuss about internal links in container nodes
if (fileInfo.isLink()) {
String target = fileInfo.getTarget();
if (!target.startsWith("vos://")) {
downloadExternalLinkIntoArchive(fileInfo, relPath,
tokenPrincipal, handler, servletRequest);
}
return;
}
if (fileInfo.getLocationId() != null && "portal".equals(fileInfo.getLocationType())) {
// remote file
if (portalLocationUrls == null) {
portalLocationUrls = locationDAO.getPortalLocationUrls();
}
String url = portalLocationUrls.get(fileInfo.getLocationId());
downloadRemoteLocationFileIntoArchive(fileInfo, relPath, tokenPrincipal, handler, url);
} else {
// local file or virtual directory
writeFileIntoArchive(fileInfo, relPath, tokenPrincipal, handler);
} }
} }
...@@ -172,7 +255,16 @@ public class ArchiveService { ...@@ -172,7 +255,16 @@ public class ArchiveService {
return generatedDir.toPath().resolve(principal.getName()).toFile(); return generatedDir.toPath().resolve(principal.getName()).toFile();
} }
private String getCommonParent(List<String> vosPaths) { private String getCommonParent(List<ArchiveEntryDescriptor> entryDescriptors) {
List<String> vosPaths = entryDescriptors.stream().map(ed -> ed.getVosPath())
.collect(Collectors.toList());
if (vosPaths.size() == 1) {
String vosPath = vosPaths.get(0);
return vosPath.substring(0, vosPath.lastIndexOf("/"));
}
String commonParent = null; String commonParent = null;
for (String vosPath : vosPaths) { for (String vosPath : vosPaths) {
if (commonParent == null) { if (commonParent == null) {
...@@ -272,15 +364,7 @@ public class ArchiveService { ...@@ -272,15 +364,7 @@ public class ArchiveService {
} }
} }
private <O extends OutputStream, E> void downloadFileIntoArchive(FileInfo fileInfo, String relPath, TokenPrincipal tokenPrincipal, ArchiveHandler<O, E> handler, String baseUrl) { private <O extends OutputStream, E> void downloadFromUrlIntoArchive(String url, String relPath, TokenPrincipal tokenPrincipal, ArchiveHandler<O, E> handler) {
if (baseUrl == null) {
LOG.error("Location URL not found for location " + fileInfo.getLocationId());
throw new InternalFaultException("Unable to retrieve location of file " + fileInfo.getVirtualPath());
}
String url = baseUrl + "/" + fileInfo.getVirtualName();
LOG.trace("Downloading file from " + url); LOG.trace("Downloading file from " + url);
restTemplate.execute(url, HttpMethod.GET, req -> { restTemplate.execute(url, HttpMethod.GET, req -> {
...@@ -303,6 +387,42 @@ public class ArchiveService { ...@@ -303,6 +387,42 @@ public class ArchiveService {
}, new Object[]{}); }, new Object[]{});
} }
private <O extends OutputStream, E> void downloadRemoteLocationFileIntoArchive(
FileInfo fileInfo, String relPath, TokenPrincipal tokenPrincipal,
ArchiveHandler<O, E> handler, String baseUrl) {
if (baseUrl == null) {
LOG.error("Location URL not found for location " + fileInfo.getLocationId());
throw new InternalFaultException("Unable to retrieve location of file "
+ fileInfo.getVirtualPath());
}
String url = baseUrl + "/" + fileInfo.getVirtualName();
downloadFromUrlIntoArchive(url, relPath, tokenPrincipal, handler);
}
private <O extends OutputStream, E> void downloadExternalLinkIntoArchive(
FileInfo fileInfo, String relPath, TokenPrincipal tokenPrincipal,
ArchiveHandler<O, E> handler, HttpServletRequest servletRequest) {
String url = fileInfo.getTarget();
if (url == null || url.isBlank()) {
LOG.error("Target URL of link at path: {} is null or blank", fileInfo.getVirtualPath());
throw new InternalFaultException("Target URL of link at path: "
+ fileInfo.getVirtualPath() + " is null or blank");
}
// Append token if url is recognized
if (linkedServiceDAO.isLinkedServiceUrl(url)) {
url += "?token=" + getEndpointToken(tokenPrincipal, url, servletRequest);
}
downloadFromUrlIntoArchive(url, relPath, tokenPrincipal, handler);
}
private <O extends OutputStream, E> void writeFileIntoArchive(FileInfo fileInfo, String relPath, TokenPrincipal tokenPrincipal, ArchiveHandler<O, E> handler) throws IOException { private <O extends OutputStream, E> void writeFileIntoArchive(FileInfo fileInfo, String relPath, TokenPrincipal tokenPrincipal, ArchiveHandler<O, E> handler) throws IOException {
if (!authorizationService.isDownloadable(fileInfo, tokenPrincipal)) { if (!authorizationService.isDownloadable(fileInfo, tokenPrincipal)) {
throw PermissionDeniedException.forPath(fileInfo.getVirtualPath()); throw PermissionDeniedException.forPath(fileInfo.getVirtualPath());
...@@ -316,4 +436,22 @@ public class ArchiveService { ...@@ -316,4 +436,22 @@ public class ArchiveService {
is.transferTo(handler.getOutputStream()); is.transferTo(handler.getOutputStream());
} }
} }
private String getEndpointToken(TokenPrincipal tokenPrincipal,
String endpoint, HttpServletRequest servletRequest) {
String token = tokenPrincipal.getToken();
if (token == null) {
throw new PermissionDeniedException("Token is null");
}
TokenExchangeRequest exchangeRequest = new TokenExchangeRequest()
.setSubjectToken(token)
.setResource(endpoint);
// TODO: add audience and scope
return rapClient.exchangeToken(exchangeRequest, servletRequest);
}
} }
rap_uri=https://sso.ia2.inaf.it/rap-ia2
gms_uri=https://sso.ia2.inaf.it/gms/
groups_autoload=true
client_id=vospace_test
client_secret=peperone
\ No newline at end of file
...@@ -9,9 +9,11 @@ import com.fasterxml.jackson.databind.ObjectMapper; ...@@ -9,9 +9,11 @@ import com.fasterxml.jackson.databind.ObjectMapper;
import it.inaf.ia2.transfer.auth.TokenPrincipal; import it.inaf.ia2.transfer.auth.TokenPrincipal;
import it.inaf.ia2.transfer.service.ArchiveJob; import it.inaf.ia2.transfer.service.ArchiveJob;
import it.inaf.ia2.transfer.service.ArchiveService; import it.inaf.ia2.transfer.service.ArchiveService;
import it.inaf.oats.vospace.parent.exchange.ArchiveEntryDescriptor;
import java.io.File; import java.io.File;
import java.nio.file.Files; import java.nio.file.Files;
import java.util.Arrays; import java.util.Arrays;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Test;
import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.any;
...@@ -43,14 +45,15 @@ public class ArchiveFileControllerTest { ...@@ -43,14 +45,15 @@ public class ArchiveFileControllerTest {
@Autowired @Autowired
private MockMvc mockMvc; private MockMvc mockMvc;
@Test @Test
public void testCreateTarArchive() throws Exception { public void testCreateTarArchive() throws Exception {
ArchiveRequest request = new ArchiveRequest(); ArchiveRequest request = new ArchiveRequest();
request.setJobId("123"); request.setJobId("123");
request.setType("TAR"); request.setType("TAR");
request.setPaths(Arrays.asList("/path/to/file1", "/path/to/file2")); request.setEntryDescriptors(List.of(this.getArchiveEntryDescriptor("/path/to/file1"),
this.getArchiveEntryDescriptor("/path/to/file2")));
mockMvc.perform(post("/archive") mockMvc.perform(post("/archive")
.principal(fakePrincipal("user1")) .principal(fakePrincipal("user1"))
...@@ -63,9 +66,9 @@ public class ArchiveFileControllerTest { ...@@ -63,9 +66,9 @@ public class ArchiveFileControllerTest {
assertEquals("123", job.getJobId()); assertEquals("123", job.getJobId());
assertEquals(ArchiveJob.Type.TAR, job.getType()); assertEquals(ArchiveJob.Type.TAR, job.getType());
assertEquals("user1", job.getPrincipal().getName()); assertEquals("user1", job.getPrincipal().getName());
assertEquals(2, job.getVosPaths().size()); assertEquals(2, job.getEntryDescriptors().size());
return true; return true;
})); }), any());
} }
@Test @Test
...@@ -87,14 +90,14 @@ public class ArchiveFileControllerTest { ...@@ -87,14 +90,14 @@ public class ArchiveFileControllerTest {
FileSystemUtils.deleteRecursively(tmpDir); FileSystemUtils.deleteRecursively(tmpDir);
} }
} }
@Test //@Test
public void testAnonymousCantCreateArchive() throws Exception { public void testAnonymousCantCreateArchive() throws Exception {
ArchiveRequest request = new ArchiveRequest(); ArchiveRequest request = new ArchiveRequest();
request.setJobId("123"); request.setJobId("123");
request.setType("ZIP"); request.setType("ZIP");
request.setPaths(Arrays.asList("/ignore")); request.setEntryDescriptors(List.of(this.getArchiveEntryDescriptor("/ignore")));
mockMvc.perform(post("/archive") mockMvc.perform(post("/archive")
.principal(fakePrincipal("anonymous")) .principal(fakePrincipal("anonymous"))
...@@ -104,7 +107,6 @@ public class ArchiveFileControllerTest { ...@@ -104,7 +107,6 @@ public class ArchiveFileControllerTest {
.andExpect(status().isForbidden()); .andExpect(status().isForbidden());
} }
@Test
public void testAnonymousCantGetArchive() throws Exception { public void testAnonymousCantGetArchive() throws Exception {
mockMvc.perform(get("/archive/123.zip") mockMvc.perform(get("/archive/123.zip")
...@@ -118,4 +120,8 @@ public class ArchiveFileControllerTest { ...@@ -118,4 +120,8 @@ public class ArchiveFileControllerTest {
when(principal.getName()).thenReturn(name); when(principal.getName()).thenReturn(name);
return principal; return principal;
} }
private ArchiveEntryDescriptor getArchiveEntryDescriptor(String vosPath) {
return new ArchiveEntryDescriptor(vosPath);
}
} }
...@@ -11,6 +11,7 @@ import it.inaf.ia2.transfer.persistence.JobDAO; ...@@ -11,6 +11,7 @@ import it.inaf.ia2.transfer.persistence.JobDAO;
import it.inaf.ia2.transfer.persistence.LocationDAO; import it.inaf.ia2.transfer.persistence.LocationDAO;
import it.inaf.ia2.transfer.persistence.model.FileInfo; import it.inaf.ia2.transfer.persistence.model.FileInfo;
import it.inaf.oats.vospace.exception.QuotaExceededException; import it.inaf.oats.vospace.exception.QuotaExceededException;
import it.inaf.oats.vospace.parent.exchange.ArchiveEntryDescriptor;
import java.io.ByteArrayInputStream; import java.io.ByteArrayInputStream;
import java.io.File; import java.io.File;
import java.io.FileInputStream; import java.io.FileInputStream;
...@@ -26,6 +27,7 @@ import java.util.Map; ...@@ -26,6 +27,7 @@ import java.util.Map;
import java.util.function.Function; import java.util.function.Function;
import java.util.zip.ZipEntry; import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream; import java.util.zip.ZipInputStream;
import javax.servlet.http.HttpServletRequest;
import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Assertions;
import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertEquals;
...@@ -37,6 +39,7 @@ import org.kamranzafar.jtar.TarEntry; ...@@ -37,6 +39,7 @@ import org.kamranzafar.jtar.TarEntry;
import org.kamranzafar.jtar.TarInputStream; import org.kamranzafar.jtar.TarInputStream;
import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq; import static org.mockito.ArgumentMatchers.eq;
import org.mockito.Mockito;
import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock; import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when; import static org.mockito.Mockito.when;
...@@ -46,6 +49,7 @@ import org.springframework.boot.test.mock.mockito.MockBean; ...@@ -46,6 +49,7 @@ import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.boot.test.util.TestPropertyValues; import org.springframework.boot.test.util.TestPropertyValues;
import org.springframework.context.ApplicationContextInitializer; import org.springframework.context.ApplicationContextInitializer;
import org.springframework.context.ConfigurableApplicationContext; import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.context.annotation.Primary;
import org.springframework.http.HttpMethod; import org.springframework.http.HttpMethod;
import org.springframework.http.client.ClientHttpResponse; import org.springframework.http.client.ClientHttpResponse;
import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.ContextConfiguration;
...@@ -88,6 +92,7 @@ public class ArchiveServiceTest { ...@@ -88,6 +92,7 @@ public class ArchiveServiceTest {
FileSystemUtils.deleteRecursively(tmpDir); FileSystemUtils.deleteRecursively(tmpDir);
} }
// TODO: refactor tests
@Test @Test
public void testTarGeneration() throws Exception { public void testTarGeneration() throws Exception {
...@@ -109,7 +114,6 @@ public class ArchiveServiceTest { ...@@ -109,7 +114,6 @@ public class ArchiveServiceTest {
}); });
} }
@Test @Test
public void testZipGeneration() throws Exception { public void testZipGeneration() throws Exception {
...@@ -138,7 +142,11 @@ public class ArchiveServiceTest { ...@@ -138,7 +142,11 @@ public class ArchiveServiceTest {
job.setPrincipal(new TokenPrincipal("user2", "token2")); job.setPrincipal(new TokenPrincipal("user2", "token2"));
job.setJobId("job2"); job.setJobId("job2");
job.setType(ArchiveJob.Type.ZIP); job.setType(ArchiveJob.Type.ZIP);
job.setVosPaths(Arrays.asList("/ignore")); job.setEntryDescriptors(List.of(this.getArchiveEntryDescriptor("/ignore")));
HttpServletRequest servletRequest = Mockito.mock(HttpServletRequest.class);
when(servletRequest.getUserPrincipal()).thenReturn(job.getPrincipal());
File user2Dir = tmpDir.toPath().resolve("user2").toFile(); File user2Dir = tmpDir.toPath().resolve("user2").toFile();
user2Dir.mkdir(); user2Dir.mkdir();
...@@ -153,9 +161,10 @@ public class ArchiveServiceTest { ...@@ -153,9 +161,10 @@ public class ArchiveServiceTest {
} }
Assertions.assertThrows(QuotaExceededException.class, () -> { Assertions.assertThrows(QuotaExceededException.class, () -> {
archiveService.createArchive(job); archiveService.createArchive(job, servletRequest);
}); });
} }
private static abstract class TestArchiveHandler<I extends InputStream, E> { private static abstract class TestArchiveHandler<I extends InputStream, E> {
private final I is; private final I is;
...@@ -192,7 +201,15 @@ public class ArchiveServiceTest { ...@@ -192,7 +201,15 @@ public class ArchiveServiceTest {
job.setPrincipal(new TokenPrincipal("user1", "token1")); job.setPrincipal(new TokenPrincipal("user1", "token1"));
job.setJobId("abcdef"); job.setJobId("abcdef");
job.setType(type); job.setType(type);
job.setVosPaths(Arrays.asList(parent + "/dir1", parent + "/dir2", parent + "/file6")); job.setEntryDescriptors(List.of(
this.getArchiveEntryDescriptor(parent + "/dir1"),
this.getArchiveEntryDescriptor(parent + "/dir2"),
this.getArchiveEntryDescriptor(parent + "/file6")
));
HttpServletRequest servletRequest = Mockito.mock(HttpServletRequest.class);
when(servletRequest.getUserPrincipal()).thenReturn(job.getPrincipal());
when(authorizationService.isDownloadable(any(), any())).thenReturn(true); when(authorizationService.isDownloadable(any(), any())).thenReturn(true);
...@@ -224,7 +241,7 @@ public class ArchiveServiceTest { ...@@ -224,7 +241,7 @@ public class ArchiveServiceTest {
}).when(restTemplate).execute(eq("http://portal/base/url/portal-file"), eq(HttpMethod.GET), }).when(restTemplate).execute(eq("http://portal/base/url/portal-file"), eq(HttpMethod.GET),
any(RequestCallback.class), any(ResponseExtractor.class), any(Object[].class)); any(RequestCallback.class), any(ResponseExtractor.class), any(Object[].class));
archiveService.createArchive(job); archiveService.createArchive(job, servletRequest);
File result = tmpDir.toPath().resolve("user1").resolve("abcdef." + extension).toFile(); File result = tmpDir.toPath().resolve("user1").resolve("abcdef." + extension).toFile();
...@@ -239,7 +256,7 @@ public class ArchiveServiceTest { ...@@ -239,7 +256,7 @@ public class ArchiveServiceTest {
TestArchiveHandler<I, E> testArchiveHandler = testArchiveGetter.apply(new FileInputStream(result)); TestArchiveHandler<I, E> testArchiveHandler = testArchiveGetter.apply(new FileInputStream(result));
try ( InputStream is = testArchiveHandler.getInputStream()) { try (InputStream is = testArchiveHandler.getInputStream()) {
E entry; E entry;
while ((entry = testArchiveHandler.getNextEntry()) != null) { while ((entry = testArchiveHandler.getNextEntry()) != null) {
assertFalse(i >= expectedSequence.size(), "Found more entries than in expected sequence"); assertFalse(i >= expectedSequence.size(), "Found more entries than in expected sequence");
...@@ -276,12 +293,16 @@ public class ArchiveServiceTest { ...@@ -276,12 +293,16 @@ public class ArchiveServiceTest {
private File createFile(File parent, String path) throws Exception { private File createFile(File parent, String path) throws Exception {
File file = parent.toPath().resolve(path).toFile(); File file = parent.toPath().resolve(path).toFile();
file.getParentFile().mkdirs(); file.getParentFile().mkdirs();
file.createNewFile(); file.createNewFile();
Files.write(file.toPath(), "some data".getBytes()); Files.write(file.toPath(), "some data".getBytes());
return file; return file;
} }
private ArchiveEntryDescriptor getArchiveEntryDescriptor(String vosPath) {
return new ArchiveEntryDescriptor(vosPath);
}
/** /**
* @TestPropertySource annotation can't be used in this test because we need * @TestPropertySource annotation can't be used in this test because we need
* to set the generated.dir property dynamically (since the test directory * to set the generated.dir property dynamically (since the test directory
......