ArchiveServiceTest.java

/*
 * This file is part of vospace-file-service
 * Copyright (C) 2021 Istituto Nazionale di Astrofisica
 * SPDX-License-Identifier: GPL-3.0-or-later
 */
package it.inaf.ia2.transfer.service;

import it.inaf.ia2.transfer.auth.TokenPrincipal;
import it.inaf.ia2.transfer.persistence.FileDAO;
import it.inaf.ia2.transfer.persistence.JobDAO;
import it.inaf.ia2.transfer.persistence.LocationDAO;
import it.inaf.ia2.transfer.persistence.model.FileInfo;
import it.inaf.oats.vospace.exception.QuotaExceededException;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.UncheckedIOException;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import javax.servlet.http.HttpServletRequest;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assertions;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.kamranzafar.jtar.TarEntry;
import org.kamranzafar.jtar.TarInputStream;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.boot.test.util.TestPropertyValues;
import org.springframework.context.ApplicationContextInitializer;
import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.http.HttpMethod;
import org.springframework.http.client.ClientHttpResponse;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.util.FileSystemUtils;
import org.springframework.web.client.RequestCallback;
import org.springframework.web.client.ResponseExtractor;
import org.springframework.web.client.RestTemplate;

@SpringBootTest
@ContextConfiguration(initializers = ArchiveServiceTest.TestPropertiesInitializer.class)
public class ArchiveServiceTest {

    @MockBean
    private JobDAO jobDAO;

    @MockBean
    private FileDAO fileDAO;

    @MockBean
    private LocationDAO locationDAO;

    @MockBean
    private RestTemplate restTemplate;
    
    @MockBean
    private HttpServletRequest servletRequest;

    @MockBean
    private AuthorizationService authorizationService;

    @Autowired
    private ArchiveService archiveService;

    private static File tmpDir;

    @BeforeAll
    public static void setUpClass() throws Exception {
        tmpDir = Files.createTempDirectory("generated").toFile();
    }

    @AfterAll
    public static void tearDownClass() throws Exception {
        FileSystemUtils.deleteRecursively(tmpDir);
    }

    @Test
    public void testTarGeneration() throws Exception {

        testArchiveGeneration(ArchiveJob.Type.TAR, "tar", is -> new TestArchiveHandler<TarInputStream, TarEntry>(new TarInputStream(is)) {
            @Override
            TarEntry getNextEntry() throws IOException {
                return getInputStream().getNextEntry();
            }

            @Override
            String getName(TarEntry entry) {
                return entry.getName();
            }

            @Override
            boolean isDirectory(TarEntry entry) {
                return entry.isDirectory();
            }
        });
    }

    
    @Test
    public void testZipGeneration() throws Exception {

        testArchiveGeneration(ArchiveJob.Type.ZIP, "zip", is -> new TestArchiveHandler<ZipInputStream, ZipEntry>(new ZipInputStream(is)) {
            @Override
            ZipEntry getNextEntry() throws IOException {
                return getInputStream().getNextEntry();
            }

            @Override
            String getName(ZipEntry entry) {
                return entry.getName();
            }

            @Override
            boolean isDirectory(ZipEntry entry) {
                return entry.isDirectory();
            }
        });
    }

    @Test
    public void testArchiveQuotaExceeded() throws Exception {

        ArchiveJob job = new ArchiveJob();
        job.setPrincipal(new TokenPrincipal("user2", "token2"));
        job.setJobId("job2");
        job.setType(ArchiveJob.Type.ZIP);
        job.setVosPaths(Arrays.asList("/ignore"));
        
        when(servletRequest.getUserPrincipal()).thenReturn(job.getPrincipal());

        File user2Dir = tmpDir.toPath().resolve("user2").toFile();
        user2Dir.mkdir();

        File fillQuotaFile = user2Dir.toPath().resolve("fillQuotaFile").toFile();

        // create a file bigger than test quota limit (20 KB)
        try (FileInputStream fis = new FileInputStream("/dev/zero");
                FileOutputStream fos = new FileOutputStream(fillQuotaFile)) {
            byte[] junk = fis.readNBytes(20 * 1024);
            fos.write(junk);
        }

        Assertions.assertThrows(QuotaExceededException.class, () -> {
            archiveService.createArchive(job, servletRequest);
        });
    }
    private static abstract class TestArchiveHandler<I extends InputStream, E> {

        private final I is;

        TestArchiveHandler(I is) {
            this.is = is;
        }

        I getInputStream() {
            return is;
        }

        abstract E getNextEntry() throws IOException;

        abstract String getName(E entry);

        abstract boolean isDirectory(E entry);
    }

    private <I extends InputStream, E> void testArchiveGeneration(ArchiveJob.Type type, String extension, Function<FileInputStream, TestArchiveHandler<I, E>> testArchiveGetter) throws Exception {

        String parent = "/path/to";

        File tmpParent = tmpDir.toPath().resolve("test1").toFile();
        File file1 = createFile(tmpParent, "2021/10/1/UUID-file1");
        File file2 = createFile(tmpParent, "2021/10/1/UUID-file2");
        File file3 = createFile(tmpParent, "2021/10/1/UUID-file3");
        File file4 = createFile(tmpParent, "2021/10/1/UUID-file4");
        File file5 = createFile(tmpParent, "2021/10/1/UUID-file5");
        File file6 = createFile(tmpParent, "2021/10/1/UUID-file6");
        File file7 = createFile(tmpParent, "2021/10/1/UUID-portal-file");

        ArchiveJob job = new ArchiveJob();
        job.setPrincipal(new TokenPrincipal("user1", "token1"));
        job.setJobId("abcdef");
        job.setType(type);
        job.setVosPaths(Arrays.asList(parent + "/dir1", parent + "/dir2", parent + "/file6"));
        
        when(servletRequest.getUserPrincipal()).thenReturn(job.getPrincipal());

        when(authorizationService.isDownloadable(any(), any())).thenReturn(true);

        List<FileInfo> fileInfos = new ArrayList<>();
        addFileInfo(fileInfos, parent + "/file6", file6);
        addDirInfo(fileInfos, parent + "/dir1");
        addDirInfo(fileInfos, parent + "/dir1/a");
        addDirInfo(fileInfos, parent + "/dir1/a/b");
        addFileInfo(fileInfos, parent + "/dir1/a/b/file1", file1);
        addFileInfo(fileInfos, parent + "/dir1/a/b/file2", file2);
        addDirInfo(fileInfos, parent + "/dir2");
        addDirInfo(fileInfos, parent + "/dir2/c");
        addFileInfo(fileInfos, parent + "/dir2/c/file3", file3);
        addFileInfo(fileInfos, parent + "/dir2/c/file4", file4);
        addDirInfo(fileInfos, parent + "/dir2/c/d");
        addFileInfo(fileInfos, parent + "/dir2/c/d/file5", file5);
        addFileInfo(fileInfos, parent + "/portal-file", file7).setLocationId(1);

        when(fileDAO.getArchiveFileInfos(any())).thenReturn(fileInfos);

        when(locationDAO.getPortalLocationUrls()).thenReturn(Map.of(1, "http://portal/base/url"));

        doAnswer(invocation -> {
            ResponseExtractor responseExtractor = invocation.getArgument(3);
            ClientHttpResponse mockedResponse = mock(ClientHttpResponse.class);
            when(mockedResponse.getBody()).thenReturn(new ByteArrayInputStream("some data".getBytes()));
            responseExtractor.extractData(mockedResponse);
            return null;
        }).when(restTemplate).execute(eq("http://portal/base/url/portal-file"), eq(HttpMethod.GET),
                any(RequestCallback.class), any(ResponseExtractor.class), any(Object[].class));

        archiveService.createArchive(job, servletRequest);

        File result = tmpDir.toPath().resolve("user1").resolve("abcdef." + extension).toFile();

        assertTrue(result.exists());

        // verify result structure
        List<String> expectedSequence = Arrays.asList("file6", "dir1/", "dir1/a/", "dir1/a/b/",
                "dir1/a/b/file1", "dir1/a/b/file2", "dir2/", "dir2/c/", "dir2/c/file3", "dir2/c/file4",
                "dir2/c/d/", "dir2/c/d/file5", "portal-file");

        int i = 0;

        TestArchiveHandler<I, E> testArchiveHandler = testArchiveGetter.apply(new FileInputStream(result));

        try ( InputStream is = testArchiveHandler.getInputStream()) {
            E entry;
            while ((entry = testArchiveHandler.getNextEntry()) != null) {
                assertFalse(i >= expectedSequence.size(), "Found more entries than in expected sequence");
                assertEquals(expectedSequence.get(i), testArchiveHandler.getName(entry));
                if (!testArchiveHandler.isDirectory(entry)) {
                    assertEquals("some data", new String(is.readAllBytes()));
                }
                i++;
            }
        } catch (IOException ex) {
            throw new UncheckedIOException(ex);
        }

        assertFalse(i < expectedSequence.size(), "Found less entries than in expected sequence");
    }

    private FileInfo addFileInfo(List<FileInfo> fileInfos, String vosPath, File file) {
        FileInfo fileInfo = new FileInfo();
        fileInfo.setActualBasePath("/");
        fileInfo.setFsPath(file.getAbsolutePath());
        fileInfo.setVirtualPath(vosPath);
        fileInfo.setVirtualName(vosPath.substring(vosPath.lastIndexOf("/") + 1));
        fileInfos.add(fileInfo);
        return fileInfo;
    }

    private FileInfo addDirInfo(List<FileInfo> fileInfos, String vosPath) {
        FileInfo fileInfo = new FileInfo();
        fileInfo.setVirtualPath(vosPath);
        fileInfo.setDirectory(true);
        fileInfos.add(fileInfo);
        return fileInfo;
    }

    private File createFile(File parent, String path) throws Exception {
        File file = parent.toPath().resolve(path).toFile();
        file.getParentFile().mkdirs();        
        file.createNewFile();
        Files.write(file.toPath(), "some data".getBytes());
        return file;
    }

    /**
     * @TestPropertySource annotation can't be used in this test because we need
     * to set the generated.dir property dynamically (since the test directory
     * is generated by the @BeforeAll method), so this inner class is used to
     * perform test property initialization.
     */
    static class TestPropertiesInitializer implements ApplicationContextInitializer<ConfigurableApplicationContext> {

        @Override
        public void initialize(ConfigurableApplicationContext configurableApplicationContext) {
            TestPropertyValues.of("generated.dir=" + tmpDir.getAbsolutePath(),
                    "generated.dir.max-size=20KB", "upload_location_id=3")
                    .applyTo(configurableApplicationContext.getEnvironment());
        }
    }
}