Merge branch 'portal_links'

src/main/java/it/inaf/oats/vospace/BaseNodeController.java

@@ -19,11 +19,11 @@ public abstract class BaseNodeController {
 
     @Autowired
     private HttpServletRequest servletRequest;
-    
+
     @Value("${vospace-authority}")
     protected String authority;
-        
-    protected String getPath() {       
+
+    protected String getPath() {
         String requestURL = servletRequest.getRequestURL().toString();
         try {
             return NodeUtils.getPathFromRequestURLString(requestURL);
@@ -35,7 +35,7 @@ public abstract class BaseNodeController {
     protected String getParentPath(String path) {
         return NodeUtils.getParentPath(path);
     }
-    
+
     protected void validateAndCheckPayloadURIConsistence(Node node) {
         // Get Node path (and validates it too)
         String decodedURIPathFromNode = URIUtils.returnVosPathFromNodeURI(node.getUri(), this.authority);
@@ -45,16 +45,27 @@ public abstract class BaseNodeController {
         if (!decodedURIPathFromNode.equals(this.getPath())) {
             throw new InvalidURIException(decodedURIPathFromNode, requestPath);
         }
-        
+
     }
-    
-    protected void validateInternalLinkNode(LinkNode linkNode) {
+
+    protected void validateLinkNode(LinkNode linkNode) {
         String target = linkNode.getTarget();
         // I validate it here to add context easily
         if (target == null) {
             throw new InvalidArgumentException("LinkNode in payload has no target element specified");
         }
 
-        URIUtils.returnVosPathFromNodeURI(linkNode.getTarget(), authority);
+        if (URIUtils.isURIInternal(target)) {
+            URIUtils.returnVosPathFromNodeURI(linkNode.getTarget(), authority);
+        } else {
+            // TODO: Let's discuss if we need to combine this validation with
+            // protocol endpoints management (URIService, ProtocolType)
+            // Let's start with http and https only for now
+            if (!(target.toLowerCase().startsWith("http://")
+                    || target.toLowerCase().startsWith("https://"))) {
+                throw new InvalidArgumentException("LinkNode target malformed or unsupported protocol: " + target);
+            }
+
+        }
     }
 }

src/main/java/it/inaf/oats/vospace/CreateNodeController.java

@@ -6,7 +6,6 @@
 package it.inaf.oats.vospace;
 
 import it.inaf.ia2.aa.data.User;
-import it.inaf.oats.vospace.exception.InvalidURIException;
 import net.ivoa.xml.vospace.v2.LinkNode;
 import net.ivoa.xml.vospace.v2.Node;
 import org.springframework.http.MediaType;
@@ -45,7 +44,7 @@ public class CreateNodeController extends BaseNodeController {
     private void validateInputNode(Node node) {
 
         if (node instanceof LinkNode) {
-            this.validateInternalLinkNode((LinkNode) node);
+            this.validateLinkNode((LinkNode) node);
         }
 
     }

src/main/java/it/inaf/oats/vospace/FileServiceClient.java

@@ -7,12 +7,19 @@ package it.inaf.oats.vospace;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
 import it.inaf.ia2.aa.data.User;
+import it.inaf.oats.vospace.datamodel.NodeUtils;
 import it.inaf.oats.vospace.datamodel.Views;
 import it.inaf.oats.vospace.exception.InvalidArgumentException;
+import it.inaf.oats.vospace.exception.NodeNotFoundException;
+import it.inaf.oats.vospace.parent.exchange.ArchiveEntryDescriptor;
+import it.inaf.oats.vospace.persistence.NodeDAO;
 import java.io.OutputStream;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.stream.Collectors;
 import javax.servlet.http.HttpServletRequest;
+import net.ivoa.xml.vospace.v2.ContainerNode;
+import net.ivoa.xml.vospace.v2.Node;
 import net.ivoa.xml.vospace.v2.Transfer;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
@@ -21,6 +28,7 @@ import org.springframework.http.HttpMethod;
 import org.springframework.http.MediaType;
 import org.springframework.stereotype.Component;
 import org.springframework.web.client.RestTemplate;
+import static org.springframework.web.servlet.mvc.method.RequestMappingInfo.paths;
 
 @Component
 public class FileServiceClient {
@@ -39,6 +47,12 @@ public class FileServiceClient {
     @Autowired
     private HttpServletRequest request;
 
+    @Autowired
+    private LinkService linkService;
+
+    @Autowired
+    private NodeDAO nodeDAO;
+
     public String startArchiveJob(Transfer transfer, String jobId) {
 
         String target = transfer.getTarget().substring("vos://".length() + authority.length());
@@ -64,9 +78,38 @@ public class FileServiceClient {
             vosPaths.add(target);
         }
 
+        // Generate descriptors
+        // Expand container nodes into direct children list
+        List<String> expandedVosPaths = new ArrayList<>();
+
+        for (String vosPath : vosPaths) {
+            Node node
+                    = nodeDAO.listNode(vosPath)
+                            .orElseThrow(() -> {
+                                throw new NodeNotFoundException(vosPath);
+                            });
+
+            if (node instanceof ContainerNode) {
+                List<Node> nodes = ((ContainerNode) node).getNodes();
+                if (nodes.isEmpty()) {
+                    expandedVosPaths.add(NodeUtils.getVosPath(node));
+                } else {
+                    expandedVosPaths.addAll(nodes
+                            .stream().map(n -> NodeUtils.getVosPath(n))
+                            .collect(Collectors.toList()));
+                }
+            } else {
+                expandedVosPaths.add(vosPath);
+            }
+
+        }
+
+        // follow links to links in vosPaths
+        List<ArchiveEntryDescriptor> entryDescriptors = linkService.followLinksForArchiveService(expandedVosPaths);
+
         ArchiveRequest archiveRequest = new ArchiveRequest();
         archiveRequest.setJobId(jobId);
-        archiveRequest.setPaths(vosPaths);
+        archiveRequest.setEntryDescriptors(entryDescriptors);
         archiveRequest.setType(archiveTypeFromViewUri(transfer.getView().getUri()));
 
         String url = fileServiceUrl + "/archive";
@@ -86,17 +129,17 @@ public class FileServiceClient {
             return res.getHeaders().getLocation().toString();
         }, new Object[]{});
     }
-    
-    public void startFileCopyJob(String sourceVosPath, 
+
+    public void startFileCopyJob(String sourceVosPath,
             String destiantionVosPath, String jobId, User user) {
-        
+
         CopyRequest copyRequest = new CopyRequest();
         copyRequest.setJobId(jobId);
         copyRequest.setSourceRootVosPath(sourceVosPath);
         copyRequest.setDestinationRootVosPath(destiantionVosPath);
-        
+
         String url = fileServiceUrl + "/copy";
-        
+
         String token = user.getAccessToken();
         restTemplate.execute(url, HttpMethod.POST, req -> {
             HttpHeaders headers = req.getHeaders();
@@ -108,10 +151,10 @@ public class FileServiceClient {
             try (OutputStream os = req.getBody()) {
                 MAPPER.writeValue(os, copyRequest);
             }
-        }, res -> {           
-           return null;
+        }, res -> {
+            return null;
         }, new Object[]{});
-        
+
     }
 
     public static class CopyRequest {
@@ -150,7 +193,7 @@ public class FileServiceClient {
 
         private String type;
         private String jobId;
-        private List<String> paths;
+        private List<ArchiveEntryDescriptor> entryDescriptors;
 
         public String getType() {
             return type;
@@ -168,12 +211,12 @@ public class FileServiceClient {
             this.jobId = jobId;
         }
 
-        public List<String> getPaths() {
-            return paths;
+        public List<ArchiveEntryDescriptor> getEntryDescriptors() {
+            return entryDescriptors;
         }
 
-        public void setPaths(List<String> paths) {
-            this.paths = paths;
+        public void setEntryDescriptors(List<ArchiveEntryDescriptor> entryDescriptors) {
+            this.entryDescriptors = entryDescriptors;
         }
     }
 

src/main/java/it/inaf/oats/vospace/LinkService.java

+/*
+ * This file is part of vospace-rest
+ * Copyright (C) 2021 Istituto Nazionale di Astrofisica
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ */
+package it.inaf.oats.vospace;
+
+import it.inaf.oats.vospace.datamodel.NodeUtils;
+import it.inaf.oats.vospace.exception.InternalFaultException;
+import it.inaf.oats.vospace.parent.exchange.ArchiveEntryDescriptor;
+import it.inaf.oats.vospace.persistence.NodeDAO;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+import net.ivoa.xml.vospace.v2.LinkNode;
+import net.ivoa.xml.vospace.v2.Node;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Service;
+
+@Service
+public class LinkService {
+
+    @Value("${vospace-authority}")
+    private String authority;
+
+    @Value("${link-max-depth}")
+    private int linkMaxDepth;
+    
+    @Autowired
+    private NodeDAO nodeDao;
+    
+    // Returns a new list = the list in argument with paths of links to links are substituted with 
+    // their actual destination vos path. Only links to external resources 
+    // (http:// ... ) should remain
+    public List<ArchiveEntryDescriptor> followLinksForArchiveService(List<String> vosPaths) {
+ 
+        List<LinkNode> linkNodesInPaths = nodeDao.returnLinkNodesInList(vosPaths);
+        
+        // No links no change
+        if(linkNodesInPaths.isEmpty())
+            return vosPaths.stream().map(p -> new ArchiveEntryDescriptor(p))
+                    .collect(Collectors.toList());
+              
+        List<String> linkVosPaths = linkNodesInPaths.stream()
+                .map(ln -> NodeUtils.getVosPath(ln)).collect(Collectors.toList());
+        
+        // Safe copy of argument        
+        List<String> resultVosPaths = new ArrayList<>(vosPaths);
+        
+        resultVosPaths.removeAll(linkVosPaths);
+        
+        // Generate descriptors from non link vospaths
+        List<ArchiveEntryDescriptor> resultDescriptors = 
+                resultVosPaths.stream().map(p -> new ArchiveEntryDescriptor(p))
+                        .collect(Collectors.toList());
+        
+        // Add descriptors from links
+        resultDescriptors.addAll(
+                linkNodesInPaths.stream().map(p -> getLinkNodeArchiveEntryDescriptor(p))
+                .collect(Collectors.toList())       
+        );              
+        
+        return resultDescriptors;
+                                    
+    }
+    
+    private ArchiveEntryDescriptor getLinkNodeArchiveEntryDescriptor(LinkNode node){
+        String vosPath = NodeUtils.getVosPath(node);        
+        String targetNodeVosPath = NodeUtils.getVosPath(this.followLink(node));
+        
+        return new ArchiveEntryDescriptor(vosPath, targetNodeVosPath);
+    }
+    
+    public Node followLink(LinkNode linkNode) {
+        return this.followLinkRecursive(linkNode, 0);
+    }
+
+    private Node followLinkRecursive(LinkNode linkNode, int depth) {
+
+        if (depth >= linkMaxDepth) {
+            throw new InternalFaultException("Max link depth reached at link node: "
+                    + NodeUtils.getVosPath(linkNode));
+        }
+
+        String linkTarget = linkNode.getTarget();
+
+        if (URIUtils.isURIInternal(linkTarget)) {
+            String targetPath = URIUtils.returnVosPathFromNodeURI(linkTarget, authority);
+
+            Optional<Node> targetNodeOpt = nodeDao.listNode(targetPath);
+            Node targetNode = targetNodeOpt.orElseThrow(() -> new InternalFaultException("Broken Link to target: " + targetPath));
+
+            if (targetNode instanceof LinkNode) {
+                return this.followLinkRecursive(linkNode, ++depth);
+            } else {
+                return targetNode;
+            }
+        } else {
+            return linkNode;
+        }
+    }
+
+}

src/main/java/it/inaf/oats/vospace/SetNodeController.java

@@ -73,7 +73,7 @@ public class SetNodeController extends BaseNodeController {
         if (node instanceof DataNode) {
             checkViews((DataNode) node, (DataNode) toBeModifiedNode);
         } else if(node instanceof LinkNode) {
-            this.validateInternalLinkNode((LinkNode) node);            
+            this.validateLinkNode((LinkNode) node);            
         }
 
         //The service SHOULD throw a HTTP 500 status code including an InternalFault fault 

src/main/java/it/inaf/oats/vospace/URIUtils.java

@@ -18,6 +18,13 @@ public class URIUtils {
     // Slashes are treated separately
     private static final Pattern FORBIDDEN_CHARS = Pattern.compile("[\\x00\\x08\\x0B\\x0C\\x0E-\\x1F" + Pattern.quote("<>?\":\\|'`*") + "]");
     private static final String SCHEME = "vos";
+    
+    public static boolean isURIInternal(String URI) {
+        if(URI == null)
+            throw new IllegalArgumentException("URI can't be null");
+        
+        return URI.toLowerCase().startsWith(SCHEME);
+    }
 
     public static String returnURIFromVosPath(String vosPath, String authority) {
         String result = null;

src/main/java/it/inaf/oats/vospace/UriService.java

@@ -19,6 +19,7 @@ import it.inaf.oats.vospace.exception.NodeNotFoundException;
 import it.inaf.oats.vospace.exception.PermissionDeniedException;
 import it.inaf.oats.vospace.exception.ProtocolNotSupportedException;
 import it.inaf.oats.vospace.exception.NodeBusyException;
+import it.inaf.oats.vospace.parent.persistence.LinkedServiceDAO;
 import it.inaf.oats.vospace.persistence.LocationDAO;
 import it.inaf.oats.vospace.persistence.NodeDAO;
 import it.inaf.oats.vospace.persistence.model.Location;
@@ -27,6 +28,7 @@ import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.stream.Collectors;
 import javax.servlet.http.HttpServletRequest;
@@ -49,15 +51,18 @@ public class UriService {
     @Value("${file-service-url}")
     private String fileServiceUrl;
 
-    @Value("${link-max-depth}")
-    private int linkMaxDepth;
-
     @Autowired
     private NodeDAO nodeDao;
+    
+    @Autowired
+    private LinkService linkService;
 
     @Autowired
     private LocationDAO locationDAO;
 
+    @Autowired
+    private LinkedServiceDAO linkedServiceDAO;
+
     @Autowired
     private HttpServletRequest servletRequest;
 
@@ -90,14 +95,15 @@ public class UriService {
         JobService.JobDirection jobDirection
                 = JobDirection.getJobDirectionEnumFromTransfer(transfer);
 
-        List<String> validProtocolUris = new ArrayList<>();
+        List<ProtocolType> validProtocolTypes = new ArrayList<>();
         switch (jobDirection) {
             case pullFromVoSpace:
+                validProtocolTypes.add(ProtocolType.HTTPSGET);
             case pullToVoSpace:
-                validProtocolUris.add("ivo://ivoa.net/vospace/core#httpget");
+                validProtocolTypes.add(ProtocolType.HTTPGET);
                 break;
             case pushToVoSpace:
-                validProtocolUris.add("ivo://ivoa.net/vospace/core#httpput");
+                validProtocolTypes.add(ProtocolType.HTTPPUT);
                 break;
 
             default:
@@ -106,15 +112,30 @@ public class UriService {
 
         List<Protocol> validProtocols
                 = transfer.getProtocols().stream()
-                        // discard invalid protocols
-                        .filter(protocol -> validProtocolUris.contains(protocol.getUri()))
+                        // discard invalid protocols by uri String
+                        .filter(protocol
+                                -> validProtocolTypes.stream().map(pt
+                                -> {
+                            return pt.getUri();
+                        })
+                                .collect(Collectors.toList())
+                                .contains(protocol.getUri()))
                         .map(p -> {
                             // set endpoints
-                            Protocol protocol = new Protocol();
-                            protocol.setUri(p.getUri());
-                            protocol.setEndpoint(getEndpoint(job, transfer));
-                            return protocol;
-                        }).collect(Collectors.toList());
+                            String endpoint = getEndpoint(job, transfer);
+                            ProtocolType pt
+                                    = ProtocolType.getProtocolTypeFromURI(p.getUri());
+
+                            if (pt.isEndpointCompliant(endpoint)) {
+                                Protocol protocol = new Protocol();
+                                protocol.setUri(p.getUri());
+                                protocol.setEndpoint(endpoint);
+                                return protocol;
+                            } else {
+                                return null;
+                            }
+                        }).filter(Objects::nonNull)
+                        .collect(Collectors.toList());
 
         if (validProtocols.isEmpty()) {
             Protocol protocol = transfer.getProtocols().get(0);
@@ -137,7 +158,7 @@ public class UriService {
                     if (!NodeUtils.checkIfReadable(node, user.getName(), user.getGroups())) {
                         throw PermissionDeniedException.forPath(relativePath);
                     }
-                    node = this.followLink((LinkNode) node);
+                    node = linkService.followLink((LinkNode) node);
                 }
             }
             return node;
@@ -168,7 +189,7 @@ public class UriService {
         JobService.JobDirection jobType
                 = JobDirection.getJobDirectionEnumFromTransfer(transfer);
         Node node = this.getEndpointNode(relativePath, jobType, user);
-               
+
         switch (jobType) {
             case pushToVoSpace:
             case pullToVoSpace:
@@ -178,7 +199,7 @@ public class UriService {
                 break;
 
             case pullFromVoSpace:
-                // Refresh relative path: it can differ in case of links
+                // Refresh relative path: it can differ in case of links followed 
                 relativePath = NodeUtils.getVosPath(node);
                 if (!NodeUtils.checkIfReadable(node, creator, groups)) {
                     throw PermissionDeniedException.forPath(relativePath);
@@ -197,25 +218,36 @@ public class UriService {
             return fileServiceClient.startArchiveJob(transfer, job.getJobId());
         }
 
-        Location location = locationDAO.getNodeLocation(relativePath).orElse(null);
+        boolean isLinkNode = node instanceof LinkNode;
 
         String endpoint;
 
-        if (location != null && location.getType() == LocationType.PORTAL) {
-            String fileName = nodeDao.getNodeOsName(relativePath);
-            endpoint = "http://" + location.getSource().getHostname() + location.getSource().getBaseUrl();
-            if (!endpoint.endsWith("/")) {
-                endpoint += "/";
+        if (isLinkNode) {
+            endpoint = ((LinkNode) node).getTarget();
+            String linkTarget = ((LinkNode) node).getTarget();
+            if (linkedServiceDAO.isLinkedServiceUrl(linkTarget)) {
+                endpoint += "?token=" + getEndpointToken(linkTarget);
             }
-            endpoint += fileName;
         } else {
-            endpoint = fileServiceUrl + urlEncodePath(relativePath);
-        }
 
-        endpoint += "?jobId=" + job.getJobId();
+            Location location = locationDAO.getNodeLocation(relativePath).orElse(null);
 
-        if (!"true".equals(NodeProperties.getNodePropertyByURI(node, NodeProperties.PUBLIC_READ_URI))) {
-            endpoint += "&token=" + getEndpointToken(fileServiceUrl + relativePath);
+            if (location != null && location.getType() == LocationType.PORTAL) {
+                String fileName = nodeDao.getNodeOsName(relativePath);
+                endpoint = "http://" + location.getSource().getHostname() + location.getSource().getBaseUrl();
+                if (!endpoint.endsWith("/")) {
+                    endpoint += "/";
+                }
+                endpoint += fileName;
+            } else {
+                endpoint = fileServiceUrl + urlEncodePath(relativePath);
+            }
+
+            endpoint += "?jobId=" + job.getJobId();
+
+            if (!"true".equals(NodeProperties.getNodePropertyByURI(node, NodeProperties.PUBLIC_READ_URI))) {
+                endpoint += "&token=" + getEndpointToken(fileServiceUrl + relativePath);
+            }
         }
 
         return endpoint;
@@ -282,26 +314,40 @@ public class UriService {
         return (Transfer) job.getJobInfo().getAny().get(0);
     }
 
-    private Node followLink(LinkNode linkNode) {
-        return this.followLinkRecursive(linkNode, 0);
-    }
+    public enum ProtocolType {
+        // Please keep the URIs in this enum UNIQUE!
+        // added a unit test to check this
+        HTTPGET("ivo://ivoa.net/vospace/core#httpget", "http"),
+        HTTPSGET("ivo://ivoa.net/vospace/core#httpsget", "https"),
+        HTTPPUT("ivo://ivoa.net/vospace/core#httpput", "http"),
+        HTTPSPUT("ivo://ivoa.net/vospace/core#httpsput", "https");
+
+        private final String uri;
+        private final String protocolString;
+
+        private ProtocolType(String uri, String protocolString) {
+            this.uri = uri;
+            this.protocolString = protocolString;
+        }
 
-    private Node followLinkRecursive(LinkNode linkNode, int depth) {
-                
-        if(depth >= linkMaxDepth) {
-            throw new InternalFaultException("Max link depth reached at link node: " 
-                    + NodeUtils.getVosPath(linkNode));
+        public String getUri() {
+            return this.uri;
         }
-        
-        String targetPath = URIUtils.returnVosPathFromNodeURI(linkNode.getTarget(), authority);
-        
-        Optional<Node> targetNodeOpt = nodeDao.listNode(targetPath);
-        Node targetNode = targetNodeOpt.orElseThrow(() -> new InternalFaultException("Broken Link to target: " + targetPath));
-        
-        if(targetNode instanceof LinkNode) {            
-            return this.followLinkRecursive(linkNode, ++depth);
-        } else {            
-            return targetNode;
+
+        public boolean isEndpointCompliant(String endpoint) {
+            return endpoint.toLowerCase()
+                    .startsWith(this.protocolString + "://");
         }
+
+        public static ProtocolType getProtocolTypeFromURI(String uri) {
+            for (ProtocolType pt : ProtocolType.values()) {
+                if (pt.getUri().equals(uri)) {
+                    return pt;
+                }
+            }
+
+            return null;
+        }
+
     }
 }

src/main/java/it/inaf/oats/vospace/persistence/NodeDAO.java

@@ -18,6 +18,7 @@ import java.sql.SQLException;
 import java.sql.Types;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
@@ -174,7 +175,7 @@ public class NodeDAO {
                 ps.setArray(++i, fromPropertyToArray(ps, NodeProperties.getNodePropertyByURI(newNode, NodeProperties.GROUP_WRITE_URI)));
                 ps.setBoolean(++i, Boolean.valueOf(NodeProperties.getNodePropertyByURI(newNode, NodeProperties.PUBLIC_READ_URI)));
                 if (isLinkNode) {
-                    ps.setString(++i, ((LinkNode) newNode).getTarget() );