Commit a80fd0ac authored by Sonia Zorba's avatar Sonia Zorba
Browse files

Merge branch 'portal_links'

parents 0b882b24 dc7531b9
Pipeline #7435 passed with stages
in 2 minutes and 32 seconds
......@@ -19,11 +19,11 @@ public abstract class BaseNodeController {
@Autowired
private HttpServletRequest servletRequest;
@Value("${vospace-authority}")
protected String authority;
protected String getPath() {
protected String getPath() {
String requestURL = servletRequest.getRequestURL().toString();
try {
return NodeUtils.getPathFromRequestURLString(requestURL);
......@@ -35,7 +35,7 @@ public abstract class BaseNodeController {
protected String getParentPath(String path) {
return NodeUtils.getParentPath(path);
}
protected void validateAndCheckPayloadURIConsistence(Node node) {
// Get Node path (and validates it too)
String decodedURIPathFromNode = URIUtils.returnVosPathFromNodeURI(node.getUri(), this.authority);
......@@ -45,16 +45,27 @@ public abstract class BaseNodeController {
if (!decodedURIPathFromNode.equals(this.getPath())) {
throw new InvalidURIException(decodedURIPathFromNode, requestPath);
}
}
protected void validateInternalLinkNode(LinkNode linkNode) {
protected void validateLinkNode(LinkNode linkNode) {
String target = linkNode.getTarget();
// I validate it here to add context easily
if (target == null) {
throw new InvalidArgumentException("LinkNode in payload has no target element specified");
}
URIUtils.returnVosPathFromNodeURI(linkNode.getTarget(), authority);
if (URIUtils.isURIInternal(target)) {
URIUtils.returnVosPathFromNodeURI(linkNode.getTarget(), authority);
} else {
// TODO: Let's discuss if we need to combine this validation with
// protocol endpoints management (URIService, ProtocolType)
// Let's start with http and https only for now
if (!(target.toLowerCase().startsWith("http://")
|| target.toLowerCase().startsWith("https://"))) {
throw new InvalidArgumentException("LinkNode target malformed or unsupported protocol: " + target);
}
}
}
}
......@@ -6,7 +6,6 @@
package it.inaf.oats.vospace;
import it.inaf.ia2.aa.data.User;
import it.inaf.oats.vospace.exception.InvalidURIException;
import net.ivoa.xml.vospace.v2.LinkNode;
import net.ivoa.xml.vospace.v2.Node;
import org.springframework.http.MediaType;
......@@ -45,7 +44,7 @@ public class CreateNodeController extends BaseNodeController {
private void validateInputNode(Node node) {
if (node instanceof LinkNode) {
this.validateInternalLinkNode((LinkNode) node);
this.validateLinkNode((LinkNode) node);
}
}
......
......@@ -7,12 +7,19 @@ package it.inaf.oats.vospace;
import com.fasterxml.jackson.databind.ObjectMapper;
import it.inaf.ia2.aa.data.User;
import it.inaf.oats.vospace.datamodel.NodeUtils;
import it.inaf.oats.vospace.datamodel.Views;
import it.inaf.oats.vospace.exception.InvalidArgumentException;
import it.inaf.oats.vospace.exception.NodeNotFoundException;
import it.inaf.oats.vospace.parent.exchange.ArchiveEntryDescriptor;
import it.inaf.oats.vospace.persistence.NodeDAO;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import javax.servlet.http.HttpServletRequest;
import net.ivoa.xml.vospace.v2.ContainerNode;
import net.ivoa.xml.vospace.v2.Node;
import net.ivoa.xml.vospace.v2.Transfer;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
......@@ -21,6 +28,7 @@ import org.springframework.http.HttpMethod;
import org.springframework.http.MediaType;
import org.springframework.stereotype.Component;
import org.springframework.web.client.RestTemplate;
import static org.springframework.web.servlet.mvc.method.RequestMappingInfo.paths;
@Component
public class FileServiceClient {
......@@ -39,6 +47,12 @@ public class FileServiceClient {
@Autowired
private HttpServletRequest request;
@Autowired
private LinkService linkService;
@Autowired
private NodeDAO nodeDAO;
public String startArchiveJob(Transfer transfer, String jobId) {
String target = transfer.getTarget().substring("vos://".length() + authority.length());
......@@ -64,9 +78,38 @@ public class FileServiceClient {
vosPaths.add(target);
}
// Generate descriptors
// Expand container nodes into direct children list
List<String> expandedVosPaths = new ArrayList<>();
for (String vosPath : vosPaths) {
Node node
= nodeDAO.listNode(vosPath)
.orElseThrow(() -> {
throw new NodeNotFoundException(vosPath);
});
if (node instanceof ContainerNode) {
List<Node> nodes = ((ContainerNode) node).getNodes();
if (nodes.isEmpty()) {
expandedVosPaths.add(NodeUtils.getVosPath(node));
} else {
expandedVosPaths.addAll(nodes
.stream().map(n -> NodeUtils.getVosPath(n))
.collect(Collectors.toList()));
}
} else {
expandedVosPaths.add(vosPath);
}
}
// follow links to links in vosPaths
List<ArchiveEntryDescriptor> entryDescriptors = linkService.followLinksForArchiveService(expandedVosPaths);
ArchiveRequest archiveRequest = new ArchiveRequest();
archiveRequest.setJobId(jobId);
archiveRequest.setPaths(vosPaths);
archiveRequest.setEntryDescriptors(entryDescriptors);
archiveRequest.setType(archiveTypeFromViewUri(transfer.getView().getUri()));
String url = fileServiceUrl + "/archive";
......@@ -86,17 +129,17 @@ public class FileServiceClient {
return res.getHeaders().getLocation().toString();
}, new Object[]{});
}
public void startFileCopyJob(String sourceVosPath,
public void startFileCopyJob(String sourceVosPath,
String destiantionVosPath, String jobId, User user) {
CopyRequest copyRequest = new CopyRequest();
copyRequest.setJobId(jobId);
copyRequest.setSourceRootVosPath(sourceVosPath);
copyRequest.setDestinationRootVosPath(destiantionVosPath);
String url = fileServiceUrl + "/copy";
String token = user.getAccessToken();
restTemplate.execute(url, HttpMethod.POST, req -> {
HttpHeaders headers = req.getHeaders();
......@@ -108,10 +151,10 @@ public class FileServiceClient {
try (OutputStream os = req.getBody()) {
MAPPER.writeValue(os, copyRequest);
}
}, res -> {
return null;
}, res -> {
return null;
}, new Object[]{});
}
public static class CopyRequest {
......@@ -150,7 +193,7 @@ public class FileServiceClient {
private String type;
private String jobId;
private List<String> paths;
private List<ArchiveEntryDescriptor> entryDescriptors;
public String getType() {
return type;
......@@ -168,12 +211,12 @@ public class FileServiceClient {
this.jobId = jobId;
}
public List<String> getPaths() {
return paths;
public List<ArchiveEntryDescriptor> getEntryDescriptors() {
return entryDescriptors;
}
public void setPaths(List<String> paths) {
this.paths = paths;
public void setEntryDescriptors(List<ArchiveEntryDescriptor> entryDescriptors) {
this.entryDescriptors = entryDescriptors;
}
}
......
/*
* This file is part of vospace-rest
* Copyright (C) 2021 Istituto Nazionale di Astrofisica
* SPDX-License-Identifier: GPL-3.0-or-later
*/
package it.inaf.oats.vospace;
import it.inaf.oats.vospace.datamodel.NodeUtils;
import it.inaf.oats.vospace.exception.InternalFaultException;
import it.inaf.oats.vospace.parent.exchange.ArchiveEntryDescriptor;
import it.inaf.oats.vospace.persistence.NodeDAO;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import java.util.stream.Collectors;
import net.ivoa.xml.vospace.v2.LinkNode;
import net.ivoa.xml.vospace.v2.Node;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
@Service
public class LinkService {
@Value("${vospace-authority}")
private String authority;
@Value("${link-max-depth}")
private int linkMaxDepth;
@Autowired
private NodeDAO nodeDao;
// Returns a new list = the list in argument with paths of links to links are substituted with
// their actual destination vos path. Only links to external resources
// (http:// ... ) should remain
public List<ArchiveEntryDescriptor> followLinksForArchiveService(List<String> vosPaths) {
List<LinkNode> linkNodesInPaths = nodeDao.returnLinkNodesInList(vosPaths);
// No links no change
if(linkNodesInPaths.isEmpty())
return vosPaths.stream().map(p -> new ArchiveEntryDescriptor(p))
.collect(Collectors.toList());
List<String> linkVosPaths = linkNodesInPaths.stream()
.map(ln -> NodeUtils.getVosPath(ln)).collect(Collectors.toList());
// Safe copy of argument
List<String> resultVosPaths = new ArrayList<>(vosPaths);
resultVosPaths.removeAll(linkVosPaths);
// Generate descriptors from non link vospaths
List<ArchiveEntryDescriptor> resultDescriptors =
resultVosPaths.stream().map(p -> new ArchiveEntryDescriptor(p))
.collect(Collectors.toList());
// Add descriptors from links
resultDescriptors.addAll(
linkNodesInPaths.stream().map(p -> getLinkNodeArchiveEntryDescriptor(p))
.collect(Collectors.toList())
);
return resultDescriptors;
}
private ArchiveEntryDescriptor getLinkNodeArchiveEntryDescriptor(LinkNode node){
String vosPath = NodeUtils.getVosPath(node);
String targetNodeVosPath = NodeUtils.getVosPath(this.followLink(node));
return new ArchiveEntryDescriptor(vosPath, targetNodeVosPath);
}
public Node followLink(LinkNode linkNode) {
return this.followLinkRecursive(linkNode, 0);
}
private Node followLinkRecursive(LinkNode linkNode, int depth) {
if (depth >= linkMaxDepth) {
throw new InternalFaultException("Max link depth reached at link node: "
+ NodeUtils.getVosPath(linkNode));
}
String linkTarget = linkNode.getTarget();
if (URIUtils.isURIInternal(linkTarget)) {
String targetPath = URIUtils.returnVosPathFromNodeURI(linkTarget, authority);
Optional<Node> targetNodeOpt = nodeDao.listNode(targetPath);
Node targetNode = targetNodeOpt.orElseThrow(() -> new InternalFaultException("Broken Link to target: " + targetPath));
if (targetNode instanceof LinkNode) {
return this.followLinkRecursive(linkNode, ++depth);
} else {
return targetNode;
}
} else {
return linkNode;
}
}
}
......@@ -73,7 +73,7 @@ public class SetNodeController extends BaseNodeController {
if (node instanceof DataNode) {
checkViews((DataNode) node, (DataNode) toBeModifiedNode);
} else if(node instanceof LinkNode) {
this.validateInternalLinkNode((LinkNode) node);
this.validateLinkNode((LinkNode) node);
}
//The service SHOULD throw a HTTP 500 status code including an InternalFault fault
......
......@@ -18,6 +18,13 @@ public class URIUtils {
// Slashes are treated separately
private static final Pattern FORBIDDEN_CHARS = Pattern.compile("[\\x00\\x08\\x0B\\x0C\\x0E-\\x1F" + Pattern.quote("<>?\":\\|'`*") + "]");
private static final String SCHEME = "vos";
public static boolean isURIInternal(String URI) {
if(URI == null)
throw new IllegalArgumentException("URI can't be null");
return URI.toLowerCase().startsWith(SCHEME);
}
public static String returnURIFromVosPath(String vosPath, String authority) {
String result = null;
......
......@@ -19,6 +19,7 @@ import it.inaf.oats.vospace.exception.NodeNotFoundException;
import it.inaf.oats.vospace.exception.PermissionDeniedException;
import it.inaf.oats.vospace.exception.ProtocolNotSupportedException;
import it.inaf.oats.vospace.exception.NodeBusyException;
import it.inaf.oats.vospace.parent.persistence.LinkedServiceDAO;
import it.inaf.oats.vospace.persistence.LocationDAO;
import it.inaf.oats.vospace.persistence.NodeDAO;
import it.inaf.oats.vospace.persistence.model.Location;
......@@ -27,6 +28,7 @@ import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.stream.Collectors;
import javax.servlet.http.HttpServletRequest;
......@@ -49,15 +51,18 @@ public class UriService {
@Value("${file-service-url}")
private String fileServiceUrl;
@Value("${link-max-depth}")
private int linkMaxDepth;
@Autowired
private NodeDAO nodeDao;
@Autowired
private LinkService linkService;
@Autowired
private LocationDAO locationDAO;
@Autowired
private LinkedServiceDAO linkedServiceDAO;
@Autowired
private HttpServletRequest servletRequest;
......@@ -90,14 +95,15 @@ public class UriService {
JobService.JobDirection jobDirection
= JobDirection.getJobDirectionEnumFromTransfer(transfer);
List<String> validProtocolUris = new ArrayList<>();
List<ProtocolType> validProtocolTypes = new ArrayList<>();
switch (jobDirection) {
case pullFromVoSpace:
validProtocolTypes.add(ProtocolType.HTTPSGET);
case pullToVoSpace:
validProtocolUris.add("ivo://ivoa.net/vospace/core#httpget");
validProtocolTypes.add(ProtocolType.HTTPGET);
break;
case pushToVoSpace:
validProtocolUris.add("ivo://ivoa.net/vospace/core#httpput");
validProtocolTypes.add(ProtocolType.HTTPPUT);
break;
default:
......@@ -106,15 +112,30 @@ public class UriService {
List<Protocol> validProtocols
= transfer.getProtocols().stream()
// discard invalid protocols
.filter(protocol -> validProtocolUris.contains(protocol.getUri()))
// discard invalid protocols by uri String
.filter(protocol
-> validProtocolTypes.stream().map(pt
-> {
return pt.getUri();
})
.collect(Collectors.toList())
.contains(protocol.getUri()))
.map(p -> {
// set endpoints
Protocol protocol = new Protocol();
protocol.setUri(p.getUri());
protocol.setEndpoint(getEndpoint(job, transfer));
return protocol;
}).collect(Collectors.toList());
String endpoint = getEndpoint(job, transfer);
ProtocolType pt
= ProtocolType.getProtocolTypeFromURI(p.getUri());
if (pt.isEndpointCompliant(endpoint)) {
Protocol protocol = new Protocol();
protocol.setUri(p.getUri());
protocol.setEndpoint(endpoint);
return protocol;
} else {
return null;
}
}).filter(Objects::nonNull)
.collect(Collectors.toList());
if (validProtocols.isEmpty()) {
Protocol protocol = transfer.getProtocols().get(0);
......@@ -137,7 +158,7 @@ public class UriService {
if (!NodeUtils.checkIfReadable(node, user.getName(), user.getGroups())) {
throw PermissionDeniedException.forPath(relativePath);
}
node = this.followLink((LinkNode) node);
node = linkService.followLink((LinkNode) node);
}
}
return node;
......@@ -168,7 +189,7 @@ public class UriService {
JobService.JobDirection jobType
= JobDirection.getJobDirectionEnumFromTransfer(transfer);
Node node = this.getEndpointNode(relativePath, jobType, user);
switch (jobType) {
case pushToVoSpace:
case pullToVoSpace:
......@@ -178,7 +199,7 @@ public class UriService {
break;
case pullFromVoSpace:
// Refresh relative path: it can differ in case of links
// Refresh relative path: it can differ in case of links followed
relativePath = NodeUtils.getVosPath(node);
if (!NodeUtils.checkIfReadable(node, creator, groups)) {
throw PermissionDeniedException.forPath(relativePath);
......@@ -197,25 +218,36 @@ public class UriService {
return fileServiceClient.startArchiveJob(transfer, job.getJobId());
}
Location location = locationDAO.getNodeLocation(relativePath).orElse(null);
boolean isLinkNode = node instanceof LinkNode;
String endpoint;
if (location != null && location.getType() == LocationType.PORTAL) {
String fileName = nodeDao.getNodeOsName(relativePath);
endpoint = "http://" + location.getSource().getHostname() + location.getSource().getBaseUrl();
if (!endpoint.endsWith("/")) {
endpoint += "/";
if (isLinkNode) {
endpoint = ((LinkNode) node).getTarget();
String linkTarget = ((LinkNode) node).getTarget();
if (linkedServiceDAO.isLinkedServiceUrl(linkTarget)) {
endpoint += "?token=" + getEndpointToken(linkTarget);
}
endpoint += fileName;
} else {
endpoint = fileServiceUrl + urlEncodePath(relativePath);
}
endpoint += "?jobId=" + job.getJobId();
Location location = locationDAO.getNodeLocation(relativePath).orElse(null);
if (!"true".equals(NodeProperties.getNodePropertyByURI(node, NodeProperties.PUBLIC_READ_URI))) {
endpoint += "&token=" + getEndpointToken(fileServiceUrl + relativePath);
if (location != null && location.getType() == LocationType.PORTAL) {
String fileName = nodeDao.getNodeOsName(relativePath);
endpoint = "http://" + location.getSource().getHostname() + location.getSource().getBaseUrl();
if (!endpoint.endsWith("/")) {
endpoint += "/";
}
endpoint += fileName;
} else {
endpoint = fileServiceUrl + urlEncodePath(relativePath);
}
endpoint += "?jobId=" + job.getJobId();
if (!"true".equals(NodeProperties.getNodePropertyByURI(node, NodeProperties.PUBLIC_READ_URI))) {
endpoint += "&token=" + getEndpointToken(fileServiceUrl + relativePath);
}
}
return endpoint;
......@@ -282,26 +314,40 @@ public class UriService {
return (Transfer) job.getJobInfo().getAny().get(0);
}
private Node followLink(LinkNode linkNode) {
return this.followLinkRecursive(linkNode, 0);
}
public enum ProtocolType {
// Please keep the URIs in this enum UNIQUE!
// added a unit test to check this
HTTPGET("ivo://ivoa.net/vospace/core#httpget", "http"),
HTTPSGET("ivo://ivoa.net/vospace/core#httpsget", "https"),
HTTPPUT("ivo://ivoa.net/vospace/core#httpput", "http"),
HTTPSPUT("ivo://ivoa.net/vospace/core#httpsput", "https");
private final String uri;
private final String protocolString;
private ProtocolType(String uri, String protocolString) {
this.uri = uri;
this.protocolString = protocolString;
}
private Node followLinkRecursive(LinkNode linkNode, int depth) {
if(depth >= linkMaxDepth) {
throw new InternalFaultException("Max link depth reached at link node: "
+ NodeUtils.getVosPath(linkNode));
public String getUri() {
return this.uri;
}
String targetPath = URIUtils.returnVosPathFromNodeURI(linkNode.getTarget(), authority);
Optional<Node> targetNodeOpt = nodeDao.listNode(targetPath);
Node targetNode = targetNodeOpt.orElseThrow(() -> new InternalFaultException("Broken Link to target: " + targetPath));
if(targetNode instanceof LinkNode) {
return this.followLinkRecursive(linkNode, ++depth);
} else {
return targetNode;
public boolean isEndpointCompliant(String endpoint) {
return endpoint.toLowerCase()
.startsWith(this.protocolString + "://");
}
public static ProtocolType getProtocolTypeFromURI(String uri) {
for (ProtocolType pt : ProtocolType.values()) {
if (pt.getUri().equals(uri)) {
return pt;
}
}
return null;
}
}
}
......@@ -18,6 +18,7 @@ import java.sql.SQLException;
import java.sql.Types;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
......@@ -174,7 +175,7 @@ public class NodeDAO {
ps.setArray(++i, fromPropertyToArray(ps, NodeProperties.getNodePropertyByURI(newNode, NodeProperties.GROUP_WRITE_URI)));
ps.setBoolean(++i, Boolean.valueOf(NodeProperties.getNodePropertyByURI(newNode, NodeProperties.PUBLIC_READ_URI)));
if (isLinkNode) {