Commit 85b0e02d authored by Sonia Zorba's avatar Sonia Zorba
Browse files

Implemented async recall of multiple files

parent 17eb0e40

vospace-ui-backend/src/main/java/it/inaf/ia2/vospace/ui/VOSpaceUiApplication.java

+6 −0
Original line number Diff line number Diff line
@@ -9,6 +9,7 @@ import org.springframework.boot.autoconfigure.SpringBootApplication; 
import org.springframework.boot.web.servlet.FilterRegistrationBean;

import org.springframework.context.annotation.Bean;

import org.springframework.core.Ordered;

import org.springframework.web.client.RestTemplate;



@SpringBootApplication

public class VOSpaceUiApplication {
@@ -47,4 +48,9 @@ public class VOSpaceUiApplication { 
        int parallelism = Math.min(0x7fff /* copied from ForkJoinPool.java */, Runtime.getRuntime().availableProcessors());

        return new ForkJoinPool(parallelism, threadFactory, null, false);

    }



    @Bean

    public RestTemplate restTemplate() {

        return new RestTemplate();

    }

}

vospace-ui-backend/src/main/java/it/inaf/ia2/vospace/ui/controller/JobController.java

+104 −4
Original line number Diff line number Diff line
@@ -2,23 +2,34 @@ package it.inaf.ia2.vospace.ui.controller; 


import it.inaf.ia2.vospace.ui.client.VOSpaceClient;

import it.inaf.ia2.vospace.ui.data.Job;

import it.inaf.ia2.vospace.ui.exception.BadRequestException;

import java.util.ArrayList;

import java.util.List;

import java.util.UUID;

import net.ivoa.xml.uws.v1.ExecutionPhase;

import net.ivoa.xml.uws.v1.JobSummary;

import net.ivoa.xml.vospace.v2.Protocol;

import net.ivoa.xml.vospace.v2.StructuredDataNode;

import net.ivoa.xml.vospace.v2.Transfer;

import net.ivoa.xml.vospace.v2.View;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.beans.factory.annotation.Value;

import org.springframework.core.io.ByteArrayResource;

import org.springframework.http.HttpEntity;

import org.springframework.http.HttpHeaders;

import org.springframework.http.HttpMethod;

import org.springframework.http.MediaType;

import org.springframework.http.ResponseEntity;

import org.springframework.util.LinkedMultiValueMap;

import org.springframework.util.MultiValueMap;

import org.springframework.web.bind.annotation.GetMapping;

import org.springframework.web.bind.annotation.PostMapping;

import org.springframework.web.bind.annotation.RequestBody;

import org.springframework.web.bind.annotation.RestController;

import org.springframework.web.client.RestTemplate;



@RestController

public class JobController {

public class JobController extends BaseController {



    @Value("${vospace-authority}")

    private String authority;
@@ -26,16 +37,26 @@ public class JobController { 
    @Autowired

    private VOSpaceClient client;



    @Autowired

    private RestTemplate restTemplate;



    @PostMapping(value = "/recall", consumes = MediaType.APPLICATION_JSON_VALUE, produces = MediaType.APPLICATION_JSON_VALUE)

    public ResponseEntity<Job> startRecallFromTapeJob(@RequestBody List<String> paths) {



        if (paths.size() != 1) {

            throw new UnsupportedOperationException();

        if (paths.isEmpty()) {

            throw new BadRequestException("Received empty list of nodes");

        }



        String target;

        if (paths.size() == 1) {

            target = "vos://" + authority + paths.get(0);

        } else {

            target = createTempListOfFilesNode(paths);

        }



        Transfer transfer = new Transfer();

        transfer.setDirection("pullToVoSpace");

        transfer.setTarget("vos://" + authority + paths.get(0));

        transfer.setTarget(target);

        Protocol protocol = new Protocol();

        protocol.setUri("ia2:tape-recall");

        transfer.getProtocols().add(protocol);
@@ -49,6 +70,85 @@ public class JobController { 
        throw new RuntimeException("Error while executing job " + job.getJobId() + ". Job phase is " + job.getPhase() + ". QUEUED expected");

    }



    private String createTempListOfFilesNode(List<String> paths) {



        StructuredDataNode dataNode = createStructuredDataNode(paths);

        client.createNode(dataNode);

        String uploadEndpoint = getTempFileEndpoint(dataNode.getUri());



        String content = String.join("\n", paths);



        upload(uploadEndpoint, content);



        return dataNode.getUri();

    }



    private StructuredDataNode createStructuredDataNode(List<String> paths) {



        List<View> views = new ArrayList<>();

        View view = new View();

        view.setUri("urn:list-of-files");

        views.add(view);



        StructuredDataNode dataNode = new StructuredDataNode();



        String parentPath = getParentPath(paths);

        String newTempFile = ".tmp-" + UUID.randomUUID().toString().replace("-", "") + ".txt";



        dataNode.setUri("vos://" + authority + parentPath + "/" + newTempFile);



        dataNode.setAccepts(views);

        dataNode.setProvides(views);



        return dataNode;

    }



    private String getParentPath(List<String> paths) {

        // All the paths have the same parent, we can choose the first for extracting the path

        String firstPath = paths.get(0);

        return firstPath.substring(0, firstPath.lastIndexOf("/"));

    }



    private String getTempFileEndpoint(String target) {



        Transfer transfer = new Transfer();

        transfer.setDirection("pushToVoSpace");

        transfer.setTarget(target);

        Protocol protocol = new Protocol();

        protocol.setUri("ivo://ivoa.net/vospace/core#httpget");

        transfer.getProtocols().add(protocol);



        return client.getFileServiceEndpoints(transfer).get(0).getEndpoint();

    }



    private void upload(String endpoint, String content) {



        HttpHeaders headers = new HttpHeaders();

        headers.setContentType(MediaType.MULTIPART_FORM_DATA);



        MultiValueMap<String, Object> parts = new LinkedMultiValueMap<>();

        parts.add("file", new MultipartFileResource(content, "list.txt"));



        HttpEntity<MultiValueMap<String, Object>> requestEntity = new HttpEntity<>(parts, headers);



        restTemplate.exchange(endpoint, HttpMethod.PUT, requestEntity, Void.class);

    }



    private class MultipartFileResource extends ByteArrayResource {



        private final String fileName;



        public MultipartFileResource(String content, String fileName) {

            super(content.getBytes());

            this.fileName = fileName;

        }



        @Override

        public String getFilename() {

            return this.fileName;

        }

    }



    @GetMapping(value = "/jobs", produces = MediaType.APPLICATION_JSON_VALUE)

    public List<Job> getJobs() {

        // TODO

vospace-ui-backend/src/main/java/it/inaf/ia2/vospace/ui/service/NodesService.java

+5 −0
Original line number Diff line number Diff line
@@ -48,6 +48,11 @@ public class NodesService { 


        NodeInfo nodeInfo = new NodeInfo(node, authority);



        if (nodeInfo.getName().startsWith(".")) {

            // hidden file

            return "";

        }



        String html = "<tr>";

        html += "<td><input type=\"checkbox\" data-node=\"" + nodeInfo.getPath() + "\" /></td>";

        html += "<td>" + getIcon(nodeInfo) + getLink(nodeInfo) + "</td>";

vospace-ui-backend/src/test/java/it/inaf/ia2/vospace/ui/controller/JobControllerTest.java

0 → 100644
+71 −0
Original line number Diff line number Diff line 
package it.inaf.ia2.vospace.ui.controller;



import it.inaf.ia2.vospace.ui.client.VOSpaceClient;

import java.util.Collections;

import net.ivoa.xml.uws.v1.ExecutionPhase;

import net.ivoa.xml.uws.v1.JobSummary;

import net.ivoa.xml.vospace.v2.Protocol;

import org.junit.jupiter.api.Test;

import static org.mockito.ArgumentMatchers.any;

import static org.mockito.ArgumentMatchers.argThat;

import static org.mockito.Mockito.when;

import org.springframework.beans.factory.annotation.Autowired;

import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc;

import org.springframework.boot.test.context.SpringBootTest;

import org.springframework.boot.test.mock.mockito.MockBean;

import org.springframework.http.MediaType;

import org.springframework.test.context.TestPropertySource;

import org.springframework.test.web.servlet.MockMvc;

import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;

import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;

import org.springframework.web.client.RestTemplate;



@SpringBootTest

@AutoConfigureMockMvc

@TestPropertySource(properties = {"vospace-authority=example.com!vospace"})

public class JobControllerTest {



    @MockBean

    private VOSpaceClient client;



    @MockBean

    private RestTemplate restTemplate;

    

    @Autowired

    private MockMvc mockMvc;



    @Test

    public void testSingleFileAsyncRecall() throws Exception {



        JobSummary job = new JobSummary();

        job.setPhase(ExecutionPhase.QUEUED);



        when(client.startTransferJob(any())).thenReturn(job);



        mockMvc.perform(post("/recall")

                .contentType(MediaType.APPLICATION_JSON)

                .content("[\"/path/to/file\"]"))

                .andExpect(status().isOk());

    }



    @Test

    public void testMultipleFilesAsyncRecall() throws Exception {



        JobSummary job = new JobSummary();

        job.setPhase(ExecutionPhase.QUEUED);



        when(client.startTransferJob(argThat(transfer -> {

            return transfer.getTarget().startsWith("vos://example.com!vospace/path/to/.tmp-");

        }))).thenReturn(job);



        Protocol protocol = new Protocol();

        protocol.setEndpoint("http://file-service/path/to/file");



        when(client.getFileServiceEndpoints(any())).thenReturn(Collections.singletonList(protocol));



        mockMvc.perform(post("/recall")

                .contentType(MediaType.APPLICATION_JSON)

                .content("[\"/path/to/file1\", \"/path/to/file2\"]"))

                .andExpect(status().isOk());

    }

}

vospace-ui-backend/src/test/java/it/inaf/ia2/vospace/ui/controller/NodesControllerTest.java

+0 −10
Original line number Diff line number Diff line 
package it.inaf.ia2.vospace.ui.controller;



import it.inaf.ia2.vospace.ui.service.NodesService;

import org.junit.jupiter.api.BeforeEach;

import org.junit.jupiter.api.Test;

import static org.mockito.ArgumentMatchers.eq;

import static org.mockito.Mockito.verify;
@@ -12,7 +11,6 @@ import org.springframework.boot.test.mock.mockito.MockBean; 
import org.springframework.test.web.servlet.MockMvc;

import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get;

import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;

import org.springframework.test.web.servlet.setup.MockMvcBuilders;



@SpringBootTest

@AutoConfigureMockMvc
@@ -21,17 +19,9 @@ public class NodesControllerTest { 
    @MockBean

    private NodesService nodesService;



    @Autowired

    private NodesController controller;



    @Autowired

    private MockMvc mockMvc;



    @BeforeEach

    public void init() {

        mockMvc = MockMvcBuilders.standaloneSetup(controller).build();

    }



    @Test

    public void testListNodesEmpty() throws Exception {