Commit 7e8b0118 authored by Nicola Fulvio Calabria's avatar Nicola Fulvio Calabria
Browse files

Added link support to archives

parent ab66d88b
......@@ -12,6 +12,7 @@ import it.inaf.ia2.transfer.service.ArchiveService;
import it.inaf.oats.vospace.exception.PermissionDeniedException;
import java.io.File;
import java.util.concurrent.CompletableFuture;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.http.HttpHeaders;
......@@ -29,6 +30,9 @@ public class ArchiveFileController extends AuthenticatedFileController {
@Autowired
private ArchiveService archiveService;
@Autowired
private HttpServletRequest servletRequest;
@Autowired
private HttpServletResponse response;
......@@ -45,7 +49,7 @@ public class ArchiveFileController extends AuthenticatedFileController {
job.setVosPaths(archiveRequest.getPaths());
CompletableFuture.runAsync(() -> {
handleFileJob(() -> archiveService.createArchive(job), job.getJobId());
handleFileJob(() -> archiveService.createArchive(job, servletRequest), job.getJobId());
});
HttpHeaders headers = new HttpHeaders();
......
......@@ -47,7 +47,7 @@ public class FileDAO {
+ "accept_views, provide_views, l.location_type, n.path <> n.relative_path AS virtual_parent,\n"
+ "(SELECT user_name FROM users WHERE user_id = creator_id) AS username, n.job_id,\n"
+ "base_path, get_os_path(n.node_id) AS os_path, ? AS vos_path, false AS is_directory,\n"
+ "type = 'link' AS is_link,\n"
+ "n.type = 'link' AS is_link, n.target,\n"
+ "fs_path \n"
+ "FROM node n\n"
+ "JOIN location l ON (n.location_id IS NOT NULL AND n.location_id = l.location_id) OR (n.location_id IS NULL AND l.location_id = ?)\n"
......@@ -180,7 +180,7 @@ public class FileDAO {
+ "(SELECT user_name FROM users WHERE user_id = n.creator_id) AS username,\n"
+ "base_path, get_os_path(n.node_id) AS os_path, get_vos_path(n.node_id) AS vos_path,\n"
+ "n.type = 'container' AS is_directory, n.name, n.location_id, n.job_id,\n"
+ "n.type = 'link' AS is_link, l.location_type\n"
+ "n.type = 'link' AS is_link, n.target, l.location_type\n"
+ "FROM node n\n"
+ "JOIN node p ON p.path @> n.path\n"
+ "LEFT JOIN location l ON l.location_id = n.location_id\n"
......@@ -213,7 +213,7 @@ public class FileDAO {
+ "(SELECT user_name FROM users WHERE user_id = n.creator_id) AS username,\n"
+ "base_path, get_os_path(n.node_id) AS os_path, get_vos_path(n.node_id) AS vos_path,\n"
+ "n.type = 'container' AS is_directory, n.name, n.location_id, n.job_id,\n"
+ "n.type = 'link' AS is_link, l.location_type\n"
+ "n.type = 'link' AS is_link, n.target, l.location_type\n"
+ "FROM node n\n"
+ "JOIN node p ON p.path @> n.path\n"
+ "LEFT JOIN location l ON l.location_id = n.location_id\n"
......@@ -289,11 +289,16 @@ public class FileDAO {
long contentLength = rs.getLong("content_length");
if (!rs.wasNull()) {
fi.setContentLength(contentLength);
}
}
fi.setContentMd5(rs.getString("content_md5"));
fi.setContentType(rs.getString("content_type"));
fi.setDirectory(rs.getBoolean("is_directory"));
fi.setLink(rs.getBoolean("is_link"));
if(rs.getBoolean("is_link")){
fi.setLink(true);
fi.setTarget(rs.getString("target"));
} else {
fi.setLink(false);
}
fi.setJobId(rs.getString("job_id"));
int locationId = rs.getInt("location_id");
if (!rs.wasNull()) {
......
......@@ -17,11 +17,12 @@ public class FileInfo {
private String fsPath;
// actualBasePath differs from base path in db due to some location type
// dependent manipulations (performed by FileDAO)
private String actualBasePath;
private String actualBasePath;
private boolean isPublic;
private boolean virtualParent;
private boolean directory;
private boolean link;
private String target;
private List<String> groupRead;
private List<String> groupWrite;
private String ownerId;
......@@ -36,6 +37,14 @@ public class FileInfo {
private String locationType;
private String jobId;
public String getTarget() {
return target;
}
public void setTarget(String target) {
this.target = target;
}
public int getNodeId() {
return nodeId;
}
......
......@@ -5,6 +5,9 @@
*/
package it.inaf.ia2.transfer.service;
import it.inaf.ia2.aa.ServletRapClient;
import it.inaf.ia2.aa.data.User;
import it.inaf.ia2.rap.client.call.TokenExchangeRequest;
import it.inaf.ia2.transfer.auth.TokenPrincipal;
import it.inaf.ia2.transfer.persistence.FileDAO;
import it.inaf.ia2.transfer.persistence.JobDAO;
......@@ -13,6 +16,7 @@ import it.inaf.ia2.transfer.persistence.model.FileInfo;
import it.inaf.oats.vospace.exception.InternalFaultException;
import it.inaf.oats.vospace.exception.PermissionDeniedException;
import it.inaf.oats.vospace.exception.QuotaExceededException;
import it.inaf.oats.vospace.parent.persistence.LinkedServiceDAO;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
......@@ -28,6 +32,7 @@ import java.util.Map;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import javax.annotation.PostConstruct;
import javax.servlet.http.HttpServletRequest;
import net.ivoa.xml.uws.v1.ExecutionPhase;
import org.kamranzafar.jtar.TarEntry;
import org.kamranzafar.jtar.TarOutputStream;
......@@ -52,6 +57,9 @@ public class ArchiveService {
@Autowired
private LocationDAO locationDAO;
@Autowired
private LinkedServiceDAO linkedServiceDAO;
@Autowired
private JobDAO jobDAO;
......@@ -61,6 +69,9 @@ public class ArchiveService {
@Autowired
private RestTemplate restTemplate;
@Autowired
private ServletRapClient rapClient;
@Value("${upload_location_id}")
private int uploadLocationId;
......@@ -84,7 +95,7 @@ public class ArchiveService {
}
}
public <O extends OutputStream, E> void createArchive(ArchiveJob job) {
public <O extends OutputStream, E> void createArchive(ArchiveJob job, HttpServletRequest servletRequest) {
jobDAO.updateJobPhase(ExecutionPhase.EXECUTING, job.getJobId());
......@@ -101,7 +112,7 @@ public class ArchiveService {
// it will be initialized only when necessary
Map<Integer, String> portalLocationUrls = null;
try ( ArchiveHandler<O, E> handler = getArchiveHandler(archiveFile, job.getType())) {
try (ArchiveHandler<O, E> handler = getArchiveHandler(archiveFile, job.getType())) {
for (FileInfo fileInfo : fileDAO.getArchiveFileInfos(job.getVosPaths())) {
......@@ -112,13 +123,21 @@ public class ArchiveService {
continue;
}
// I expect only external links
// local links have been resolved before calling this endpoint
if (fileInfo.isLink()) {
downloadExternalLinkIntoArchive(fileInfo, relPath,
job.getPrincipal(), handler, servletRequest);
continue;
}
if (fileInfo.getLocationId() != null && "portal".equals(fileInfo.getLocationType())) {
// remote file
if (portalLocationUrls == null) {
portalLocationUrls = locationDAO.getPortalLocationUrls();
}
String url = portalLocationUrls.get(fileInfo.getLocationId());
downloadFileIntoArchive(fileInfo, relPath, job.getPrincipal(), handler, url);
downloadRemoteLocationFileIntoArchive(fileInfo, relPath, job.getPrincipal(), handler, url);
} else {
// local file or virtual directory
writeFileIntoArchive(fileInfo, relPath, job.getPrincipal(), handler);
......@@ -272,15 +291,7 @@ public class ArchiveService {
}
}
private <O extends OutputStream, E> void downloadFileIntoArchive(FileInfo fileInfo, String relPath, TokenPrincipal tokenPrincipal, ArchiveHandler<O, E> handler, String baseUrl) {
if (baseUrl == null) {
LOG.error("Location URL not found for location " + fileInfo.getLocationId());
throw new InternalFaultException("Unable to retrieve location of file " + fileInfo.getVirtualPath());
}
String url = baseUrl + "/" + fileInfo.getVirtualName();
private <O extends OutputStream, E> void downloadFromUrlIntoArchive(String url, String relPath, TokenPrincipal tokenPrincipal, ArchiveHandler<O, E> handler) {
LOG.trace("Downloading file from " + url);
restTemplate.execute(url, HttpMethod.GET, req -> {
......@@ -290,10 +301,10 @@ public class ArchiveService {
}
}, res -> {
File tmpFile = Files.createTempFile("download", null).toFile();
try ( FileOutputStream os = new FileOutputStream(tmpFile)) {
try (FileOutputStream os = new FileOutputStream(tmpFile)) {
res.getBody().transferTo(os);
handler.putNextEntry(tmpFile, relPath);
try ( FileInputStream is = new FileInputStream(tmpFile)) {
try (FileInputStream is = new FileInputStream(tmpFile)) {
is.transferTo(handler.getOutputStream());
}
} finally {
......@@ -303,6 +314,43 @@ public class ArchiveService {
}, new Object[]{});
}
private <O extends OutputStream, E> void downloadRemoteLocationFileIntoArchive(
FileInfo fileInfo, String relPath, TokenPrincipal tokenPrincipal,
ArchiveHandler<O, E> handler, String baseUrl) {
if (baseUrl == null) {
LOG.error("Location URL not found for location " + fileInfo.getLocationId());
throw new InternalFaultException("Unable to retrieve location of file "
+ fileInfo.getVirtualPath());
}
String url = baseUrl + "/" + fileInfo.getVirtualName();
downloadFromUrlIntoArchive(url, relPath, tokenPrincipal, handler);
}
private <O extends OutputStream, E> void downloadExternalLinkIntoArchive(
FileInfo fileInfo, String relPath, TokenPrincipal tokenPrincipal,
ArchiveHandler<O, E> handler, HttpServletRequest servletRequest) {
String url = fileInfo.getTarget();
if (url == null || url.isBlank()) {
LOG.error("Target URL of link at path: {} is null or blank", fileInfo.getVirtualPath());
throw new InternalFaultException("Target URL of link at path: "
+ fileInfo.getVirtualPath() + " is null or blank");
}
// Append token if url is recognized
if (linkedServiceDAO.isLinkedServiceUrl(url)) {
url += "?token=" + getEndpointToken(tokenPrincipal, url, servletRequest);
}
downloadFromUrlIntoArchive(url, relPath, tokenPrincipal, handler);
}
private <O extends OutputStream, E> void writeFileIntoArchive(FileInfo fileInfo, String relPath, TokenPrincipal tokenPrincipal, ArchiveHandler<O, E> handler) throws IOException {
if (!authorizationService.isDownloadable(fileInfo, tokenPrincipal)) {
throw PermissionDeniedException.forPath(fileInfo.getVirtualPath());
......@@ -311,9 +359,27 @@ public class ArchiveService {
File file = new File(fileInfo.getFilePath());
LOG.trace("Adding file " + file.getAbsolutePath() + " to tar archive");
try ( InputStream is = new FileInputStream(file)) {
try (InputStream is = new FileInputStream(file)) {
handler.putNextEntry(file, relPath);
is.transferTo(handler.getOutputStream());
}
}
private String getEndpointToken(TokenPrincipal tokenPrincipal,
String endpoint, HttpServletRequest servletRequest) {
String token = tokenPrincipal.getToken();
if (token == null) {
throw new PermissionDeniedException("Token is null");
}
TokenExchangeRequest exchangeRequest = new TokenExchangeRequest()
.setSubjectToken(token)
.setResource(endpoint);
// TODO: add audience and scope
return rapClient.exchangeToken(exchangeRequest, servletRequest);
}
}
......@@ -65,7 +65,7 @@ public class ArchiveFileControllerTest {
assertEquals("user1", job.getPrincipal().getName());
assertEquals(2, job.getVosPaths().size());
return true;
}));
}), any());
}
@Test
......
......@@ -26,6 +26,7 @@ import java.util.Map;
import java.util.function.Function;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import javax.servlet.http.HttpServletRequest;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions;
import static org.junit.jupiter.api.Assertions.assertEquals;
......@@ -69,6 +70,9 @@ public class ArchiveServiceTest {
@MockBean
private RestTemplate restTemplate;
@MockBean
private HttpServletRequest servletRequest;
@MockBean
private AuthorizationService authorizationService;
......@@ -139,6 +143,8 @@ public class ArchiveServiceTest {
job.setJobId("job2");
job.setType(ArchiveJob.Type.ZIP);
job.setVosPaths(Arrays.asList("/ignore"));
when(servletRequest.getUserPrincipal()).thenReturn(job.getPrincipal());
File user2Dir = tmpDir.toPath().resolve("user2").toFile();
user2Dir.mkdir();
......@@ -153,7 +159,7 @@ public class ArchiveServiceTest {
}
Assertions.assertThrows(QuotaExceededException.class, () -> {
archiveService.createArchive(job);
archiveService.createArchive(job, servletRequest);
});
}
private static abstract class TestArchiveHandler<I extends InputStream, E> {
......@@ -193,6 +199,8 @@ public class ArchiveServiceTest {
job.setJobId("abcdef");
job.setType(type);
job.setVosPaths(Arrays.asList(parent + "/dir1", parent + "/dir2", parent + "/file6"));
when(servletRequest.getUserPrincipal()).thenReturn(job.getPrincipal());
when(authorizationService.isDownloadable(any(), any())).thenReturn(true);
......@@ -224,7 +232,7 @@ public class ArchiveServiceTest {
}).when(restTemplate).execute(eq("http://portal/base/url/portal-file"), eq(HttpMethod.GET),
any(RequestCallback.class), any(ResponseExtractor.class), any(Object[].class));
archiveService.createArchive(job);
archiveService.createArchive(job, servletRequest);
File result = tmpDir.toPath().resolve("user1").resolve("abcdef." + extension).toFile();
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment