Commit b4e11d28 authored by Nicola Fulvio Calabria's avatar Nicola Fulvio Calabria
Browse files

Commented out or deleted code about portal locations

parent d37aac36

src/main/java/it/inaf/ia2/transfer/persistence/LocationDAO.java

deleted100644 → 0
+0 −44
Original line number Diff line number Diff line 
/*

 * This file is part of vospace-file-service

 * Copyright (C) 2021 Istituto Nazionale di Astrofisica

 * SPDX-License-Identifier: GPL-3.0-or-later

 */

package it.inaf.ia2.transfer.persistence;



import java.util.HashMap;

import java.util.Map;

import javax.sql.DataSource;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.jdbc.core.JdbcTemplate;

import org.springframework.stereotype.Repository;



@Repository

public class LocationDAO {



    private final JdbcTemplate jdbcTemplate;



    @Autowired

    public LocationDAO(DataSource fileCatalogDatasource) {

        this.jdbcTemplate = new JdbcTemplate(fileCatalogDatasource);

    }



    public Map<Integer, String> getPortalLocationUrls() {



        String sql = "SELECT location_id, hostname, base_url\n"

                + "FROM location l\n"

                + "JOIN storage s ON s.storage_id = l.storage_dest_id\n"

                + "WHERE location_type = 'portal'";



        return jdbcTemplate.query(sql, rs -> {

            Map<Integer, String> locationUrls = new HashMap<>();

            while (rs.next()) {

                int locationId = rs.getInt("location_id");

                String hostname = rs.getString("hostname");

                String baseUrl = rs.getString("base_url");

                String url = "http://" + hostname + baseUrl;

                locationUrls.put(locationId, url);

            }

            return locationUrls;

        });

    }

}

src/main/java/it/inaf/ia2/transfer/service/ArchiveService.java

+13 −49
Original line number Diff line number Diff line
@@ -6,12 +6,10 @@ 
package it.inaf.ia2.transfer.service;



import it.inaf.ia2.aa.ServletRapClient;

import it.inaf.ia2.aa.data.User;

import it.inaf.ia2.rap.client.call.TokenExchangeRequest;

import it.inaf.ia2.transfer.auth.TokenPrincipal;

import it.inaf.ia2.transfer.persistence.FileDAO;

import it.inaf.ia2.transfer.persistence.JobDAO;

import it.inaf.ia2.transfer.persistence.LocationDAO;

import it.inaf.ia2.transfer.persistence.model.FileInfo;

import it.inaf.oats.vospace.exception.InternalFaultException;

import it.inaf.oats.vospace.exception.PermissionDeniedException;
@@ -57,9 +55,6 @@ public class ArchiveService { 
    @Autowired

    private FileDAO fileDAO;



    @Autowired

    private LocationDAO locationDAO;



    @Autowired

    private LinkedServiceDAO linkedServiceDAO;
@@ -75,9 +70,6 @@ public class ArchiveService { 
    @Autowired

    private ServletRapClient rapClient;



    @Value("${upload_location_id}")

    private int uploadLocationId;



    // Directory containing temporary files generated by jobs.

    @Value("${generated.dir}")

    private String generatedDirString;
@@ -109,13 +101,11 @@ public class ArchiveService { 


            List<ArchiveEntryDescriptor> entryDescriptors = job.getEntryDescriptors();



            String commonParent = getCommonParent(entryDescriptors);



            // support directory used to generate folder inside tar files (path is redefined each time by TarEntry class)

            File supportDir = Files.createTempDirectory("dir").toFile();



            try (ArchiveHandler<O, E> handler = getArchiveHandler(archiveFile, job.getType())) {

                fillArchive(entryDescriptors, commonParent, supportDir,

                fillArchive(entryDescriptors, supportDir,

                        job.getPrincipal(), servletRequest, handler);

            } finally {

                FileSystemUtils.deleteRecursively(supportDir);
@@ -127,13 +117,10 @@ public class ArchiveService { 
    }



    private <O extends OutputStream, E> void fillArchive(

            List<ArchiveEntryDescriptor> entryDescriptors, String commonParent,

            List<ArchiveEntryDescriptor> entryDescriptors,

            File supportDir, TokenPrincipal tokenPrincipal,

            HttpServletRequest servletRequest, ArchiveHandler<O, E> handler) throws IOException {



        // it will be initialized only when necessary

        Map<Integer, String> portalLocationUrls = null;



        List<ArchiveEntryDescriptor> noTargetEntryDescriptors

                = entryDescriptors.stream().filter(ed -> !ed.isPointingToAnotherNode())

                        .collect(Collectors.toList());
@@ -143,12 +130,14 @@ public class ArchiveService { 
                .map(ed -> ed.getVosPath())

                .collect(Collectors.toList());

        

        String commonParent = getCommonParent(entryDescriptors);



        if (!vosPaths.isEmpty()) {

            for (FileInfo fileInfo : fileDAO.getArchiveFileInfos(vosPaths)) {



                String relPath = fileInfo.getVirtualPath().substring(commonParent.length());



                this.insertEntryIntoArchive(fileInfo, supportDir, relPath, tokenPrincipal, portalLocationUrls, servletRequest, handler);

                this.insertEntryIntoArchive(fileInfo, supportDir, relPath, tokenPrincipal, servletRequest, handler);



            }

        }
@@ -175,7 +164,7 @@ public class ArchiveService { 
                for (String vosPath : linkVosPaths) {

                    String relPath = vosPath.substring(commonParent.length());



                    this.insertEntryIntoArchive(fileInfo, supportDir, relPath, tokenPrincipal, portalLocationUrls, servletRequest, handler);

                    this.insertEntryIntoArchive(fileInfo, supportDir, relPath, tokenPrincipal, servletRequest, handler);

                }

            }

        }
@@ -183,8 +172,7 @@ public class ArchiveService { 


    private <O extends OutputStream, E> void insertEntryIntoArchive(

            FileInfo fileInfo, File supportDir, String relPath,

            TokenPrincipal tokenPrincipal, Map<Integer, String> portalLocationUrls,

            HttpServletRequest servletRequest, ArchiveHandler<O, E> handler)

            TokenPrincipal tokenPrincipal, HttpServletRequest servletRequest, ArchiveHandler<O, E> handler)

            throws IOException {

        if (fileInfo.isDirectory()) {

            handler.putNextEntry(supportDir, relPath);
@@ -203,17 +191,8 @@ public class ArchiveService { 
            return;

        }

        

        if (fileInfo.getLocationId() != null && "portal".equals(fileInfo.getLocationType())) {

            // remote file

            if (portalLocationUrls == null) {

                portalLocationUrls = locationDAO.getPortalLocationUrls();

            }

            String url = portalLocationUrls.get(fileInfo.getLocationId());

            downloadRemoteLocationFileIntoArchive(fileInfo, relPath, tokenPrincipal, handler, url);

        } else {

            // local file or virtual directory

        writeFileIntoArchive(fileInfo, relPath, tokenPrincipal, handler);

        }



    }



    private File getArchiveFile(ArchiveJob job) throws IOException {
@@ -387,21 +366,6 @@ public class ArchiveService { 
        }, new Object[]{});

    }



    private <O extends OutputStream, E> void downloadRemoteLocationFileIntoArchive(

            FileInfo fileInfo, String relPath, TokenPrincipal tokenPrincipal,

            ArchiveHandler<O, E> handler, String baseUrl) {



        if (baseUrl == null) {

            LOG.error("Location URL not found for location " + fileInfo.getLocationId());

            throw new InternalFaultException("Unable to retrieve location of file "

                    + fileInfo.getVirtualPath());

        }



        String url = baseUrl + "/" + fileInfo.getVirtualName();



        downloadFromUrlIntoArchive(url, relPath, tokenPrincipal, handler);

    }



    private <O extends OutputStream, E> void downloadExternalLinkIntoArchive(

            FileInfo fileInfo, String relPath, TokenPrincipal tokenPrincipal,

            ArchiveHandler<O, E> handler, HttpServletRequest servletRequest) {

src/main/java/it/inaf/ia2/transfer/service/FileCopyService.java

+2 −60
Original line number Diff line number Diff line
@@ -7,23 +7,17 @@ package it.inaf.ia2.transfer.service; 


import it.inaf.ia2.transfer.auth.TokenPrincipal;

import it.inaf.ia2.transfer.persistence.FileDAO;

import it.inaf.ia2.transfer.persistence.LocationDAO;

import it.inaf.ia2.transfer.persistence.model.FileInfo;

import it.inaf.oats.vospace.exception.InternalFaultException;

import it.inaf.oats.vospace.exception.NodeNotFoundException;

import it.inaf.oats.vospace.exception.PermissionDeniedException;

import it.inaf.oats.vospace.exception.QuotaExceededException;

import java.io.File;

import java.io.InputStream;

import java.util.List;

import java.util.Map;

import java.util.Optional;

import org.slf4j.Logger;

import org.slf4j.LoggerFactory;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.beans.factory.annotation.Value;

import org.springframework.http.HttpHeaders;

import org.springframework.http.HttpMethod;

import org.springframework.stereotype.Service;

import org.springframework.web.client.RestTemplate;
@@ -35,9 +29,6 @@ public class FileCopyService { 
    @Autowired

    private FileDAO fileDAO;



    @Autowired

    private LocationDAO locationDAO;



    @Autowired

    private AuthorizationService authorizationService;
@@ -67,7 +58,6 @@ public class FileCopyService { 
            throw new NodeNotFoundException(sourceRootVosPath);

        }





        // Set location of destinations to this file service update location

        // before retrieving file infos

        fileDAO.setBranchLocationId(destinationRootVosPath, jobId, uploadLocationId);
@@ -101,9 +91,6 @@ public class FileCopyService { 
            String destinationRootVosPath,

            TokenPrincipal principal) {



        // it will be initialized only when necessary

        Map<Integer, String> portalLocationUrls = null;



        for (FileInfo destinationFileInfo : destinationFileInfos) {

            LOG.trace("Processing {} destination", destinationFileInfo.getVirtualPath());

            // Cycle on files only
@@ -134,24 +121,9 @@ public class FileCopyService { 
                    }

                }



                if (sourceFileInfo.getLocationId() != null && "portal".equals(sourceFileInfo.getLocationType())) {

                    // remote file

                    if (portalLocationUrls == null) {

                        portalLocationUrls = locationDAO.getPortalLocationUrls();

                    }

                    String url = portalLocationUrls.get(sourceFileInfo.getLocationId());

                    // download file to destination disk path

                    this.downloadFileToDisk(sourceFileInfo,

                            destinationFileInfo,

                            principal, url, remainingQuota);



                } else {

                    // local file

                this.copyLocalFile(sourceFileInfo, destinationFileInfo, principal, remainingQuota);



            }



            }

        }

    }
@@ -168,36 +140,6 @@ public class FileCopyService { 


    }



    private void downloadFileToDisk(FileInfo sourceFileInfo,

            FileInfo destinationFileInfo, TokenPrincipal tokenPrincipal, String baseUrl, Long remainingQuota) {



        if (baseUrl == null) {

            LOG.error("Location URL not found for location " + sourceFileInfo.getLocationId());

            throw new InternalFaultException("Unable to retrieve location of file " + sourceFileInfo.getVirtualPath());

        }



        String url = baseUrl + "/" + sourceFileInfo.getVirtualName();



        LOG.trace("Downloading file from {}", url);



        restTemplate.execute(url, HttpMethod.GET, req -> {

            HttpHeaders headers = req.getHeaders();

            if (tokenPrincipal.getToken() != null) {

                headers.setBearerAuth(tokenPrincipal.getToken());

            }

        }, res -> {



            try (InputStream in = res.getBody()) {

                putFileService.storeFileFromInputStream(sourceFileInfo, destinationFileInfo, in, remainingQuota);

            } catch (Exception ex) {

                // outFile.delete();

                throw new RuntimeException(ex);

            }



            return null;

        }, new Object[]{});

    }



    private void copyLocalFile(FileInfo sourceFileInfo,

            FileInfo destinationFileInfo, TokenPrincipal tokenPrincipal, Long remainingQuota) {

src/test/java/it/inaf/ia2/transfer/persistence/LocationDAOTest.java

deleted100644 → 0
+0 −41
Original line number Diff line number Diff line 
/*

 * This file is part of vospace-file-service

 * Copyright (C) 2021 Istituto Nazionale di Astrofisica

 * SPDX-License-Identifier: GPL-3.0-or-later

 */

package it.inaf.ia2.transfer.persistence;



import java.util.Map;

import javax.sql.DataSource;

import static org.junit.jupiter.api.Assertions.assertEquals;

import org.junit.jupiter.api.BeforeEach;

import org.junit.jupiter.api.Test;

import org.junit.jupiter.api.extension.ExtendWith;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.test.context.ContextConfiguration;

import org.springframework.test.context.TestPropertySource;

import org.springframework.test.context.junit.jupiter.SpringExtension;



@ExtendWith(SpringExtension.class)

@ContextConfiguration(classes = {DataSourceConfig.class})

@TestPropertySource(locations = "classpath:test.properties")

public class LocationDAOTest {



    @Autowired

    private DataSource dataSource;



    private LocationDAO dao;



    @BeforeEach

    public void init() {

        dao = new LocationDAO(dataSource);

    }



    @Test

    public void testGetPortalLocationUrls() {

        Map<Integer, String> map = dao.getPortalLocationUrls();



        assertEquals(1, map.size());

        assertEquals("http://archive.lbto.org/files/lbt", map.get(4));

    }

}

src/test/java/it/inaf/ia2/transfer/service/ArchiveServiceTest.java

+5 −12
Original line number Diff line number Diff line
@@ -8,7 +8,6 @@ package it.inaf.ia2.transfer.service; 
import it.inaf.ia2.transfer.auth.TokenPrincipal;

import it.inaf.ia2.transfer.persistence.FileDAO;

import it.inaf.ia2.transfer.persistence.JobDAO;

import it.inaf.ia2.transfer.persistence.LocationDAO;

import it.inaf.ia2.transfer.persistence.model.FileInfo;

import it.inaf.oats.vospace.exception.QuotaExceededException;

import it.inaf.oats.vospace.parent.exchange.ArchiveEntryDescriptor;
@@ -49,7 +48,6 @@ import org.springframework.boot.test.mock.mockito.MockBean; 
import org.springframework.boot.test.util.TestPropertyValues;

import org.springframework.context.ApplicationContextInitializer;

import org.springframework.context.ConfigurableApplicationContext;

import org.springframework.context.annotation.Primary;

import org.springframework.http.HttpMethod;

import org.springframework.http.client.ClientHttpResponse;

import org.springframework.test.context.ContextConfiguration;
@@ -68,9 +66,6 @@ public class ArchiveServiceTest { 
    @MockBean

    private FileDAO fileDAO;



    @MockBean

    private LocationDAO locationDAO;



    @MockBean

    private RestTemplate restTemplate;
@@ -195,7 +190,6 @@ public class ArchiveServiceTest { 
        File file4 = createFile(tmpParent, "2021/10/1/UUID-file4");

        File file5 = createFile(tmpParent, "2021/10/1/UUID-file5");

        File file6 = createFile(tmpParent, "2021/10/1/UUID-file6");

        File file7 = createFile(tmpParent, "2021/10/1/UUID-portal-file");



        ArchiveJob job = new ArchiveJob();

        job.setPrincipal(new TokenPrincipal("user1", "token1"));
@@ -226,12 +220,10 @@ public class ArchiveServiceTest { 
        addFileInfo(fileInfos, parent + "/dir2/c/file4", file4);

        addDirInfo(fileInfos, parent + "/dir2/c/d");

        addFileInfo(fileInfos, parent + "/dir2/c/d/file5", file5);        

        addFileInfo(fileInfos, parent + "/portal-file", file7).setLocationId(1);



        when(fileDAO.getArchiveFileInfos(any())).thenReturn(fileInfos);

        

        when(locationDAO.getPortalLocationUrls()).thenReturn(Map.of(1, "http://portal/base/url"));



        /*

        doAnswer(invocation -> {

            ResponseExtractor responseExtractor = invocation.getArgument(3);

            ClientHttpResponse mockedResponse = mock(ClientHttpResponse.class);
@@ -240,6 +232,7 @@ public class ArchiveServiceTest { 
            return null;

        }).when(restTemplate).execute(eq("http://portal/base/url/portal-file"), eq(HttpMethod.GET),

                any(RequestCallback.class), any(ResponseExtractor.class), any(Object[].class));

        */



        archiveService.createArchive(job, servletRequest);
@@ -250,7 +243,7 @@ public class ArchiveServiceTest { 
        // verify result structure

        List<String> expectedSequence = Arrays.asList("file6", "dir1/", "dir1/a/", "dir1/a/b/",

                "dir1/a/b/file1", "dir1/a/b/file2", "dir2/", "dir2/c/", "dir2/c/file3", "dir2/c/file4",

                "dir2/c/d/", "dir2/c/d/file5", "portal-file");

                "dir2/c/d/", "dir2/c/d/file5");



        int i = 0;