Commit 7e8b0118 authored by Nicola Fulvio Calabria's avatar Nicola Fulvio Calabria
Browse files

Added link support to archives

parent ab66d88b

src/main/java/it/inaf/ia2/transfer/controller/ArchiveFileController.java

+5 −1
Original line number Diff line number Diff line
@@ -12,6 +12,7 @@ import it.inaf.ia2.transfer.service.ArchiveService; 
import it.inaf.oats.vospace.exception.PermissionDeniedException;

import java.io.File;

import java.util.concurrent.CompletableFuture;

import javax.servlet.http.HttpServletRequest;

import javax.servlet.http.HttpServletResponse;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.http.HttpHeaders;
@@ -30,6 +31,9 @@ public class ArchiveFileController extends AuthenticatedFileController { 
    @Autowired

    private ArchiveService archiveService;

    

    @Autowired

    private HttpServletRequest servletRequest;



    @Autowired

    private HttpServletResponse response;
@@ -45,7 +49,7 @@ public class ArchiveFileController extends AuthenticatedFileController { 
        job.setVosPaths(archiveRequest.getPaths());



        CompletableFuture.runAsync(() -> {

            handleFileJob(() -> archiveService.createArchive(job), job.getJobId());

            handleFileJob(() -> archiveService.createArchive(job, servletRequest), job.getJobId());

        });



        HttpHeaders headers = new HttpHeaders();

src/main/java/it/inaf/ia2/transfer/persistence/FileDAO.java

+10 −5
Original line number Diff line number Diff line
@@ -47,7 +47,7 @@ public class FileDAO { 
                + "accept_views, provide_views, l.location_type, n.path <> n.relative_path AS virtual_parent,\n"

                + "(SELECT user_name FROM users WHERE user_id = creator_id) AS username, n.job_id,\n"

                + "base_path, get_os_path(n.node_id) AS os_path, ? AS vos_path, false AS is_directory,\n"

                + "type = 'link' AS is_link,\n"

                + "n.type = 'link' AS is_link, n.target,\n"

                + "fs_path \n"

                + "FROM node n\n"

                + "JOIN location l ON (n.location_id IS NOT NULL AND n.location_id = l.location_id) OR (n.location_id IS NULL AND l.location_id = ?)\n"
@@ -180,7 +180,7 @@ public class FileDAO { 
                + "(SELECT user_name FROM users WHERE user_id = n.creator_id) AS username,\n"

                + "base_path, get_os_path(n.node_id) AS os_path, get_vos_path(n.node_id) AS vos_path,\n"

                + "n.type = 'container' AS is_directory, n.name, n.location_id, n.job_id,\n"

                + "n.type = 'link' AS is_link, l.location_type\n"

                + "n.type = 'link' AS is_link, n.target, l.location_type\n"

                + "FROM node n\n"

                + "JOIN node p ON p.path @> n.path\n"

                + "LEFT JOIN location l ON l.location_id = n.location_id\n"
@@ -213,7 +213,7 @@ public class FileDAO { 
                + "(SELECT user_name FROM users WHERE user_id = n.creator_id) AS username,\n"

                + "base_path, get_os_path(n.node_id) AS os_path, get_vos_path(n.node_id) AS vos_path,\n"

                + "n.type = 'container' AS is_directory, n.name, n.location_id, n.job_id,\n"

                + "n.type = 'link' AS is_link, l.location_type\n"

                + "n.type = 'link' AS is_link, n.target, l.location_type\n"

                + "FROM node n\n"

                + "JOIN node p ON p.path @> n.path\n"

                + "LEFT JOIN location l ON l.location_id = n.location_id\n"
@@ -293,7 +293,12 @@ public class FileDAO { 
        fi.setContentMd5(rs.getString("content_md5"));

        fi.setContentType(rs.getString("content_type"));

        fi.setDirectory(rs.getBoolean("is_directory"));

        fi.setLink(rs.getBoolean("is_link"));

        if(rs.getBoolean("is_link")){

            fi.setLink(true);

            fi.setTarget(rs.getString("target"));

        } else {

            fi.setLink(false);

        }

        fi.setJobId(rs.getString("job_id"));

        int locationId = rs.getInt("location_id");

        if (!rs.wasNull()) {

src/main/java/it/inaf/ia2/transfer/persistence/model/FileInfo.java

+10 −1
Original line number Diff line number Diff line
@@ -22,6 +22,7 @@ public class FileInfo { 
    private boolean virtualParent;

    private boolean directory;

    private boolean link;

    private String target;

    private List<String> groupRead;

    private List<String> groupWrite;

    private String ownerId;
@@ -36,6 +37,14 @@ public class FileInfo { 
    private String locationType;

    private String jobId;



    public String getTarget() {

        return target;

    }



    public void setTarget(String target) {

        this.target = target;

    }



    public int getNodeId() {

        return nodeId;

    }

src/main/java/it/inaf/ia2/transfer/service/ArchiveService.java

+81 −15
Original line number Diff line number Diff line
@@ -5,6 +5,9 @@ 
 */

package it.inaf.ia2.transfer.service;



import it.inaf.ia2.aa.ServletRapClient;

import it.inaf.ia2.aa.data.User;

import it.inaf.ia2.rap.client.call.TokenExchangeRequest;

import it.inaf.ia2.transfer.auth.TokenPrincipal;

import it.inaf.ia2.transfer.persistence.FileDAO;

import it.inaf.ia2.transfer.persistence.JobDAO;
@@ -13,6 +16,7 @@ import it.inaf.ia2.transfer.persistence.model.FileInfo; 
import it.inaf.oats.vospace.exception.InternalFaultException;

import it.inaf.oats.vospace.exception.PermissionDeniedException;

import it.inaf.oats.vospace.exception.QuotaExceededException;

import it.inaf.oats.vospace.parent.persistence.LinkedServiceDAO;

import java.io.BufferedOutputStream;

import java.io.File;

import java.io.FileInputStream;
@@ -28,6 +32,7 @@ import java.util.Map; 
import java.util.zip.ZipEntry;

import java.util.zip.ZipOutputStream;

import javax.annotation.PostConstruct;

import javax.servlet.http.HttpServletRequest;

import net.ivoa.xml.uws.v1.ExecutionPhase;

import org.kamranzafar.jtar.TarEntry;

import org.kamranzafar.jtar.TarOutputStream;
@@ -53,6 +58,9 @@ public class ArchiveService { 
    @Autowired

    private LocationDAO locationDAO;

    

    @Autowired

    private LinkedServiceDAO linkedServiceDAO;



    @Autowired

    private JobDAO jobDAO;
@@ -62,6 +70,9 @@ public class ArchiveService { 
    @Autowired

    private RestTemplate restTemplate;

    

    @Autowired

    private ServletRapClient rapClient;



    @Value("${upload_location_id}")

    private int uploadLocationId;
@@ -84,7 +95,7 @@ public class ArchiveService { 
        }

    }



    public <O extends OutputStream, E> void createArchive(ArchiveJob job) {

    public <O extends OutputStream, E> void createArchive(ArchiveJob job, HttpServletRequest servletRequest) {



        jobDAO.updateJobPhase(ExecutionPhase.EXECUTING, job.getJobId());
@@ -112,13 +123,21 @@ public class ArchiveService { 
                        continue;

                    }



                    // I expect only external links

                    // local links have been resolved before calling this endpoint                    

                    if (fileInfo.isLink()) {

                        downloadExternalLinkIntoArchive(fileInfo, relPath, 

                                job.getPrincipal(), handler, servletRequest);

                        continue;

                    }



                    if (fileInfo.getLocationId() != null && "portal".equals(fileInfo.getLocationType())) {

                        // remote file

                        if (portalLocationUrls == null) {

                            portalLocationUrls = locationDAO.getPortalLocationUrls();

                        }

                        String url = portalLocationUrls.get(fileInfo.getLocationId());

                        downloadFileIntoArchive(fileInfo, relPath, job.getPrincipal(), handler, url);

                        downloadRemoteLocationFileIntoArchive(fileInfo, relPath, job.getPrincipal(), handler, url);

                    } else {

                        // local file or virtual directory

                        writeFileIntoArchive(fileInfo, relPath, job.getPrincipal(), handler);
@@ -272,15 +291,7 @@ public class ArchiveService { 
        }

    }



    private <O extends OutputStream, E> void downloadFileIntoArchive(FileInfo fileInfo, String relPath, TokenPrincipal tokenPrincipal, ArchiveHandler<O, E> handler, String baseUrl) {



        if (baseUrl == null) {

            LOG.error("Location URL not found for location " + fileInfo.getLocationId());

            throw new InternalFaultException("Unable to retrieve location of file " + fileInfo.getVirtualPath());

        }



        String url = baseUrl + "/" + fileInfo.getVirtualName();



    private <O extends OutputStream, E> void downloadFromUrlIntoArchive(String url, String relPath, TokenPrincipal tokenPrincipal, ArchiveHandler<O, E> handler) {

        LOG.trace("Downloading file from " + url);



        restTemplate.execute(url, HttpMethod.GET, req -> {
@@ -303,6 +314,43 @@ public class ArchiveService { 
        }, new Object[]{});

    }



    private <O extends OutputStream, E> void downloadRemoteLocationFileIntoArchive(

            FileInfo fileInfo, String relPath, TokenPrincipal tokenPrincipal,

            ArchiveHandler<O, E> handler, String baseUrl) {



        if (baseUrl == null) {

            LOG.error("Location URL not found for location " + fileInfo.getLocationId());

            throw new InternalFaultException("Unable to retrieve location of file "

                    + fileInfo.getVirtualPath());

        }



        String url = baseUrl + "/" + fileInfo.getVirtualName();



        downloadFromUrlIntoArchive(url, relPath, tokenPrincipal, handler);

    }



    private <O extends OutputStream, E> void downloadExternalLinkIntoArchive(

            FileInfo fileInfo, String relPath, TokenPrincipal tokenPrincipal,

            ArchiveHandler<O, E> handler, HttpServletRequest servletRequest) {



        String url = fileInfo.getTarget();



        if (url == null || url.isBlank()) {

            LOG.error("Target URL of link at path: {} is null or blank", fileInfo.getVirtualPath());

            throw new InternalFaultException("Target URL of link at path: "

                    + fileInfo.getVirtualPath() + " is null or blank");

        }

        

        // Append token if url is recognized

        

        if (linkedServiceDAO.isLinkedServiceUrl(url)) {

                url += "?token=" + getEndpointToken(tokenPrincipal, url, servletRequest);

            }

        

        downloadFromUrlIntoArchive(url, relPath, tokenPrincipal, handler);

        

    }



    private <O extends OutputStream, E> void writeFileIntoArchive(FileInfo fileInfo, String relPath, TokenPrincipal tokenPrincipal, ArchiveHandler<O, E> handler) throws IOException {

        if (!authorizationService.isDownloadable(fileInfo, tokenPrincipal)) {

            throw PermissionDeniedException.forPath(fileInfo.getVirtualPath());
@@ -316,4 +364,22 @@ public class ArchiveService { 
            is.transferTo(handler.getOutputStream());

        }

    }

    

    private String getEndpointToken(TokenPrincipal tokenPrincipal, 

            String endpoint, HttpServletRequest servletRequest) {



        String token = tokenPrincipal.getToken();



        if (token == null) {

            throw new PermissionDeniedException("Token is null");

        }



        TokenExchangeRequest exchangeRequest = new TokenExchangeRequest()

                .setSubjectToken(token)

                .setResource(endpoint);



        // TODO: add audience and scope

        return rapClient.exchangeToken(exchangeRequest, servletRequest);

    }

    

}

src/test/java/it/inaf/ia2/transfer/controller/ArchiveFileControllerTest.java

+1 −1
Original line number Diff line number Diff line
@@ -65,7 +65,7 @@ public class ArchiveFileControllerTest { 
            assertEquals("user1", job.getPrincipal().getName());

            assertEquals(2, job.getVosPaths().size());

            return true;

        }));

        }), any());

    }



    @Test