Tar archive generation: handled directory structure (4a576fcf) · Commits · VOSpace INAF / vospace-file-service

src/main/java/it/inaf/ia2/transfer/persistence/FileDAO.java

+4 −2

Original line number	Diff line number	Diff line
		@@ -42,7 +42,7 @@ public class FileDAO {
		+ "content_type, content_encoding, content_length, content_md5,\n"
		+ "accept_views, provide_views, l.location_type, n.path <> n.relative_path AS virtual_parent,\n"
		+ "(SELECT user_name FROM users WHERE user_id = creator_id) AS username,\n"
		+ "base_path, get_os_path(n.node_id) AS os_path, ? AS vos_path\n"
		+ "base_path, get_os_path(n.node_id) AS os_path, ? AS vos_path, false AS is_directory\n"
		+ "FROM node n\n"
		+ "JOIN location l ON (n.location_id IS NOT NULL AND n.location_id = l.location_id) OR (n.location_id IS NULL AND l.location_id = ?)\n"
		+ "LEFT JOIN storage s ON s.storage_id = l.storage_dest_id\n"
		@@ -132,7 +132,8 @@ public class FileDAO {
		+ "content_type, content_encoding, content_length, content_md5,\n"
		+ "accept_views, provide_views, l.location_type, n.path <> n.relative_path AS virtual_parent,\n"
		+ "(SELECT user_name FROM users WHERE user_id = creator_id) AS username,\n"
		+ "base_path, get_os_path(n.node_id) AS os_path, get_vos_path(n.node_id) AS vos_path\n"
		+ "base_path, get_os_path(n.node_id) AS os_path, get_vos_path(n.node_id) AS vos_path,\n"
		+ "type = 'container' AS is_directory\n"
		+ "FROM node n\n"
		+ "JOIN location l ON (n.location_id IS NOT NULL AND n.location_id = l.location_id) OR (n.location_id IS NULL AND l.location_id = ?)\n"
		+ "LEFT JOIN storage s ON s.storage_id = l.storage_dest_id\n"
		@@ -172,6 +173,7 @@ public class FileDAO {
		fi.setContentLength(rs.getLong("content_length"));
		fi.setContentMd5(rs.getString("content_md5"));
		fi.setContentType(rs.getString("content_type"));
		fi.setDirectory(rs.getBoolean("is_directory"));

		fillOsPath(fi, rs);

src/main/java/it/inaf/ia2/transfer/persistence/model/FileInfo.java

+9 −0

Original line number	Diff line number	Diff line
		@@ -14,6 +14,7 @@ public class FileInfo {
		private String virtualPath;
		private boolean isPublic;
		private boolean virtualParent;
		private boolean directory;
		private List<String> groupRead;
		private List<String> groupWrite;
		private String ownerId;
		@@ -89,6 +90,14 @@ public class FileInfo {
		this.isPublic = isPublic;
		}

		public boolean isDirectory() {
		return directory;
		}

		public void setDirectory(boolean directory) {
		this.directory = directory;
		}

		public boolean hasVirtualParent() {
		return virtualParent;
		}

src/main/java/it/inaf/ia2/transfer/service/ArchiveService.java

+41 −4

Original line number	Diff line number	Diff line
		@@ -14,6 +14,8 @@ import java.io.FileInputStream;
		import java.io.FileOutputStream;
		import java.io.IOException;
		import java.io.InputStream;
		import java.nio.file.Files;
		import java.util.List;
		import org.kamranzafar.jtar.TarEntry;
		import org.kamranzafar.jtar.TarOutputStream;
		import org.slf4j.Logger;
		@@ -21,6 +23,7 @@ import org.slf4j.LoggerFactory;
		import org.springframework.beans.factory.annotation.Autowired;
		import org.springframework.beans.factory.annotation.Value;
		import org.springframework.stereotype.Service;
		import org.springframework.util.FileSystemUtils;

		@Service
		public class ArchiveService {
		@@ -66,12 +69,23 @@ public class ArchiveService {
		throw new IllegalStateException("Unable to create file " + archiveFile.getAbsolutePath());
		}

		String commonParent = getCommonParent(job.getVosPaths());
		// support directory used to generate folder inside tar files (path is redefined each time by TarEntry class)
		File supportDir = Files.createTempDirectory("dir").toFile();

		try ( TarOutputStream tos = new TarOutputStream(
		new BufferedOutputStream(new FileOutputStream(archiveFile)))) {

		for (FileInfo fileInfo : fileDAO.getArchiveFileInfos(job.getVosPaths())) {
		// TODO: handle different locations

		String relPath = fileInfo.getVirtualPath().substring(commonParent.length());

		if (fileInfo.isDirectory()) {
		tos.putNextEntry(new TarEntry(supportDir, relPath));
		continue;
		}

		// TODO: handle different locations
		if (!authorizationService.isDownloadable(fileInfo, job.getPrincipal())) {
		// TODO: proper exception type
		throw new RuntimeException("Unauthorized");
		@@ -79,8 +93,10 @@ public class ArchiveService {

		File file = new File(fileInfo.getOsPath());
		LOG.trace("Adding file " + file.getAbsolutePath() + " to tar archive");
		writeFileIntoTarArchive(file, tos);
		writeFileIntoTarArchive(file, relPath, tos);
		}
		} finally {
		FileSystemUtils.deleteRecursively(supportDir);
		}
		// TODO: update job status

		@@ -89,8 +105,29 @@ public class ArchiveService {
		}
		}

		private void writeFileIntoTarArchive(File file, TarOutputStream tos) throws IOException {
		TarEntry tarEntry = new TarEntry(file, file.getName());
		private String getCommonParent(List<String> vosPaths) {
		String commonParent = null;
		for (String vosPath : vosPaths) {
		if (commonParent == null) {
		commonParent = vosPath;
		} else {
		StringBuilder newCommonParent = new StringBuilder();
		boolean same = true;
		for (int i = 0; same && i < Math.min(commonParent.length(), vosPath.length()); i++) {
		if (commonParent.charAt(i) == vosPath.charAt(i)) {
		newCommonParent.append(commonParent.charAt(i));
		} else {
		same = false;
		}
		}
		commonParent = newCommonParent.toString();
		}
		}
		return commonParent;
		}

		private void writeFileIntoTarArchive(File file, String path, TarOutputStream tos) throws IOException {
		TarEntry tarEntry = new TarEntry(file, path);

		try ( InputStream is = new FileInputStream(file)) {
		tos.putNextEntry(tarEntry);

src/test/java/it/inaf/ia2/transfer/service/ArchiveServiceTest.java

+57 −4

Original line number	Diff line number	Diff line
		@@ -9,16 +9,21 @@ import it.inaf.ia2.transfer.auth.TokenPrincipal;
		import it.inaf.ia2.transfer.persistence.FileDAO;
		import it.inaf.ia2.transfer.persistence.model.FileInfo;
		import java.io.File;
		import java.io.FileInputStream;
		import java.nio.file.Files;
		import java.util.ArrayList;
		import java.util.Arrays;
		import java.util.List;
		import org.junit.jupiter.api.AfterAll;
		import static org.junit.jupiter.api.Assertions.assertEquals;
		import static org.junit.jupiter.api.Assertions.assertFalse;
		import static org.junit.jupiter.api.Assertions.assertTrue;
		import org.junit.jupiter.api.BeforeAll;
		import org.junit.jupiter.api.BeforeEach;
		import org.junit.jupiter.api.Test;
		import org.junit.jupiter.api.extension.ExtendWith;
		import org.kamranzafar.jtar.TarEntry;
		import org.kamranzafar.jtar.TarInputStream;
		import static org.mockito.ArgumentMatchers.any;
		import org.mockito.Mock;
		import static org.mockito.Mockito.when;
		@@ -59,6 +64,8 @@ public class ArchiveServiceTest {
		@Test
		public void testTarGeneration() throws Exception {

		String parent = "/path/to";

		File tmpParent = tmpDir.toPath().resolve("test1").toFile();
		File file1 = createFile(tmpParent, "dir1/a/b/file1");
		File file2 = createFile(tmpParent, "dir1/a/b/file2");
		@@ -71,21 +78,65 @@ public class ArchiveServiceTest {
		job.setPrincipal(new TokenPrincipal("user123", "token123"));
		job.setJobId("abcdef");
		job.setType(ArchiveJob.Type.TAR);
		job.setVosPaths(Arrays.asList("/path/to/file6"));//"/path/to/dir1", "/path/to/dir2",
		job.setVosPaths(Arrays.asList(parent + "/dir1", parent + "/dir2", parent + "/file6"));

		when(authorizationService.isDownloadable(any(), any())).thenReturn(true);

		List<FileInfo> fileInfos = new ArrayList<>();
		FileInfo fileInfo = new FileInfo();
		fileInfo.setOsPath(file6.getAbsolutePath());
		fileInfos.add(fileInfo);
		addFileInfo(fileInfos, parent + "/file6", file6);
		addDirInfo(fileInfos, parent + "/dir1");
		addDirInfo(fileInfos, parent + "/dir1/a");
		addDirInfo(fileInfos, parent + "/dir1/a/b");
		addFileInfo(fileInfos, parent + "/dir1/a/b/file1", file1);
		addFileInfo(fileInfos, parent + "/dir1/a/b/file2", file2);
		addDirInfo(fileInfos, parent + "/dir2");
		addDirInfo(fileInfos, parent + "/dir2/c");
		addFileInfo(fileInfos, parent + "/dir2/c/file3", file3);
		addFileInfo(fileInfos, parent + "/dir2/c/file4", file4);
		addDirInfo(fileInfos, parent + "/dir2/c/d");
		addFileInfo(fileInfos, parent + "/dir2/c/d/file5", file5);

		when(fileDAO.getArchiveFileInfos(any())).thenReturn(fileInfos);

		archiveService.createArchive(job);

		File result = tmpDir.toPath().resolve("user123").resolve("abcdef.tar").toFile();

		assertTrue(result.exists());

		// verify result structure
		List<String> expectedSequence = Arrays.asList("file6", "dir1/", "dir1/a/", "dir1/a/b/",
		"dir1/a/b/file1", "dir1/a/b/file2", "dir2/", "dir2/c/", "dir2/c/file3", "dir2/c/file4",
		"dir2/c/d/", "dir2/c/d/file5");

		int i = 0;
		try ( TarInputStream tis = new TarInputStream(new FileInputStream(result))) {
		TarEntry entry;
		while ((entry = tis.getNextEntry()) != null) {
		assertFalse(i >= expectedSequence.size(), "Found more entries than in expected sequence");
		assertEquals(expectedSequence.get(i), entry.getName());
		if (!entry.isDirectory()) {
		assertEquals("some data", new String(tis.readAllBytes()));
		}
		i++;
		}
		}

		assertFalse(i < expectedSequence.size(), "Found less entries than in expected sequence");
		}

		private void addFileInfo(List<FileInfo> fileInfos, String vosPath, File file) {
		FileInfo fileInfo = new FileInfo();
		fileInfo.setOsPath(file.getAbsolutePath());
		fileInfo.setVirtualPath(vosPath);
		fileInfos.add(fileInfo);
		}

		private void addDirInfo(List<FileInfo> fileInfos, String vosPath) {
		FileInfo fileInfo = new FileInfo();
		fileInfo.setVirtualPath(vosPath);
		fileInfo.setDirectory(true);
		fileInfos.add(fileInfo);
		}

		private File createFile(File parent, String path) throws Exception {
		@@ -94,6 +145,8 @@ public class ArchiveServiceTest {
		for (int i = 0; i < files.length; i++) {
		File file = parent.toPath().resolve(files[i]).toFile();
		if (i == files.length - 1) {
		// test os_path different from vos_path
		file.renameTo(file.getParentFile().toPath().resolve(file.getName() + "-renamed").toFile());
		file.createNewFile();
		Files.write(file.toPath(), "some data".getBytes());
		return file;