Skip to content
Commits on Source (14)
......@@ -25,7 +25,7 @@ import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RestController;
@RestController
public class ArchiveFileController extends FileController {
public class ArchiveFileController extends AuthenticatedFileController {
@Autowired
private ArchiveService archiveService;
......@@ -63,13 +63,9 @@ public class ArchiveFileController extends FileController {
FileResponseUtil.getFileResponse(response, file);
}
private TokenPrincipal getPrincipal() {
TokenPrincipal principal = (TokenPrincipal) request.getUserPrincipal();
if ("anonymous".equals(principal.getName())) {
throw new PermissionDeniedException("Tar/Zip archive generation not allowed to anonymous users");
}
return principal;
@Override
protected String getCustomAuthErrorMessage() {
return "Tar/Zip archive generation not allowed to anonymous users";
}
}
/*
* This file is part of vospace-file-service
* Copyright (C) 2021 Istituto Nazionale di Astrofisica
* SPDX-License-Identifier: GPL-3.0-or-later
*/
package it.inaf.ia2.transfer.controller;
import it.inaf.ia2.transfer.auth.TokenPrincipal;
import it.inaf.oats.vospace.exception.PermissionDeniedException;
public abstract class AuthenticatedFileController extends FileController {
protected TokenPrincipal getPrincipal() {
TokenPrincipal principal = (TokenPrincipal) request.getUserPrincipal();
if ("anonymous".equals(principal.getName())) {
throw new PermissionDeniedException(this.getCustomAuthErrorMessage());
}
return principal;
}
protected abstract String getCustomAuthErrorMessage();
}
/*
* This file is part of vospace-file-service
* Copyright (C) 2021 Istituto Nazionale di Astrofisica
* SPDX-License-Identifier: GPL-3.0-or-later
*/
package it.inaf.ia2.transfer.controller;
import it.inaf.ia2.transfer.persistence.FileDAO;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RestController;
import org.springframework.beans.factory.annotation.Autowired;
import it.inaf.ia2.transfer.service.FileCopyService;
import it.inaf.oats.vospace.exception.InvalidArgumentException;
import java.util.concurrent.CompletableFuture;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import it.inaf.ia2.transfer.auth.TokenPrincipal;
@RestController
public class CopyController extends AuthenticatedFileController {
private static final Logger LOG = LoggerFactory.getLogger(CopyController.class);
@Autowired
private FileCopyService copyService;
@Autowired
private FileDAO fileDao;
@PostMapping(value = "/copy", consumes = MediaType.APPLICATION_JSON_VALUE)
public ResponseEntity<?> copyFiles(@RequestBody CopyRequest copyRequest) {
String jobId = copyRequest.getJobId();
if (jobId == null) {
throw new InvalidArgumentException("Job Id cannot be null");
} else if (!jobDAO.isJobExisting(jobId)) {
throw new InvalidArgumentException("Job " + jobId + " not found");
}
LOG.debug("copyFiles called from jobId {}", jobId);
TokenPrincipal principal = getPrincipal();
// need to make a completable future start
CompletableFuture.runAsync(() -> {
handleFileJob(() -> {
try {
copyService.copyFiles(copyRequest.getSourceRootVosPath(),
copyRequest.getDestinationRootVosPath(), copyRequest.getJobId(),
principal);
} finally {
// TODO: cleanup code to remove unpopulated nodes in case
// of failure?
fileDao.releaseBusyNodesByJobId(jobId);
}
}, jobId);
});
return ResponseEntity.ok(
copyRequest.getJobId() + " copy task accepted by File Service"
);
}
@Override
protected String getCustomAuthErrorMessage() {
return "File Copy not allowed to anonymous users";
}
}
/*
* This file is part of vospace-file-service
* Copyright (C) 2021 Istituto Nazionale di Astrofisica
* SPDX-License-Identifier: GPL-3.0-or-later
*/
package it.inaf.ia2.transfer.controller;
public class CopyRequest {
private String jobId;
private String sourceRootVosPath;
private String destinationRootVosPath;
public String getJobId() {
return jobId;
}
public void setJobId(String jobId) {
this.jobId = jobId;
}
public String getSourceRootVosPath() {
return sourceRootVosPath;
}
public void setSourceRootVosPath(String sourceRootVosPath) {
this.sourceRootVosPath = sourceRootVosPath;
}
public String getDestinationRootVosPath() {
return destinationRootVosPath;
}
public void setDestinationRootVosPath(String destinationRootVosPath) {
this.destinationRootVosPath = destinationRootVosPath;
}
}
......@@ -26,7 +26,7 @@ public abstract class FileController {
protected HttpServletRequest request;
@Autowired
private JobDAO jobDAO;
protected JobDAO jobDAO;
public String getPath() {
......
......@@ -7,21 +7,15 @@ package it.inaf.ia2.transfer.controller;
import it.inaf.ia2.transfer.persistence.model.FileInfo;
import it.inaf.ia2.transfer.persistence.FileDAO;
import it.inaf.ia2.transfer.persistence.JobDAO;
import it.inaf.ia2.transfer.service.PutFileService;
import it.inaf.oats.vospace.exception.InvalidArgumentException;
import it.inaf.oats.vospace.exception.NodeNotFoundException;
import it.inaf.oats.vospace.exception.QuotaExceededException;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.security.MessageDigest;
import java.io.UncheckedIOException;
import java.security.NoSuchAlgorithmException;
import java.util.Optional;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.servlet.http.HttpServletRequest;
import javax.xml.bind.DatatypeConverter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
......@@ -41,7 +35,7 @@ public class PutFileController extends FileController {
private FileDAO fileDAO;
@Autowired
private JobDAO jobDAO;
private PutFileService putFileService;
@PutMapping("/**")
public void putFile(@RequestHeader(value = HttpHeaders.CONTENT_ENCODING, required = false) String contentEncoding,
......@@ -62,132 +56,45 @@ public class PutFileController extends FileController {
}
handleFileJob(() -> {
Optional<FileInfo> optFileInfo = fileDAO.getFileInfo(path);
if (optFileInfo.isPresent()) {
FileInfo fileInfo = optFileInfo.get();
String parentPath = fileInfo.getVirtualPath().substring(0, fileInfo.getVirtualPath().lastIndexOf("/"));
Long remainingQuota = fileDAO.getRemainingQuota(parentPath);
// if MultipartFile provides file size it is possible to check
// quota limit before reading the stream
if (remainingQuota != null && file != null && file.getSize() > remainingQuota) {
throw new QuotaExceededException("Path: " + fileInfo.getVirtualPath());
}
if (file != null) {
fileInfo.setContentType(file.getContentType());
}
fileInfo.setContentEncoding(contentEncoding);
try (InputStream in = file != null ? file.getInputStream() : request.getInputStream()) {
storeGenericFile(fileInfo, in, jobId, remainingQuota);
} catch (IOException | NoSuchAlgorithmException ex) {
throw new RuntimeException(ex);
int nodes = fileDAO.setBusy(path, jobId);
if (nodes == 1) {
try {
FileInfo fileInfo
= fileDAO.getFileInfo(path).orElseThrow(
// This can happen only if some code ignores busy state
// and deletes the node
() -> {
throw new NodeNotFoundException(path);
});
String parentPath = FileInfo.getVosParentPath(fileInfo);
Long remainingQuota = fileDAO.getRemainingQuota(parentPath);
// if MultipartFile provides file size it is possible to check
// quota limit before reading the stream
if (remainingQuota != null && file != null && file.getSize() > remainingQuota) {
throw new QuotaExceededException("Path: " + fileInfo.getVirtualPath());
}
if (file != null) {
fileInfo.setContentType(file.getContentType());
}
fileInfo.setContentEncoding(contentEncoding);
try (InputStream in = file != null ? file.getInputStream() : request.getInputStream()) {
putFileService.storeFileFromInputStream(fileInfo, in, remainingQuota);
} catch (IOException ex) {
throw new UncheckedIOException(ex);
}
} finally {
fileDAO.setBusy(path, null);
}
} else {
throw new NodeNotFoundException(path);
}
}, jobId);
}
private void storeGenericFile(FileInfo fileInfo, InputStream is, String jobId, Long remainingQuota) throws IOException, NoSuchAlgorithmException {
File file = new File(fileInfo.getOsPath());
/**
* This block must be synchronized, to avoid concurrency issues when
* multiple files are uploaded to a new folder in parallel.
*/
synchronized (this) {
if (!file.getParentFile().exists()) {
if (!file.getParentFile().mkdirs()) {
throw new IllegalStateException("Unable to create parent folder: " + file.getParentFile().getAbsolutePath());
}
}
}
String originalFileName = file.getName();
file = getEmptyFile(file, 1);
if (!originalFileName.equals(file.getName())) {
fileDAO.setOsName(fileInfo.getNodeId(), file.getName());
}
try {
fileDAO.setBusy(fileInfo.getNodeId(), jobId);
Files.copy(is, file.toPath());
if (fileInfo.getContentType() == null) {
fileInfo.setContentType(Files.probeContentType(file.toPath()));
}
Long fileSize = Files.size(file.toPath());
// Quota limit is checked again to handle cases where MultipartFile is not used
if (remainingQuota != null && fileSize > remainingQuota) {
file.delete();
throw new QuotaExceededException("Path: " + fileInfo.getVirtualPath());
}
String md5Checksum = makeMD5Checksum(file);
fileDAO.updateFileAttributes(fileInfo.getNodeId(),
fileInfo.getContentType(),
fileInfo.getContentEncoding(),
fileSize,
md5Checksum);
} finally {
fileDAO.setBusy(fileInfo.getNodeId(), null);
}
}
/**
* Handles duplicate file uploads generating a new non existent path. This
* is necessary in some edge cases, like when a file has been renamed in
* VOSpace only but the original file on disk still has the old name or if a
* file has been marked for deletion and a file with the same name is
* uploaded before the cleanup.
*/
private File getEmptyFile(File file, int index) {
if (file.exists()) {
String fileName = file.getName();
String nameWithoutExtension;
String extension = null;
if (fileName.contains(".")) {
nameWithoutExtension = fileName.substring(0, fileName.lastIndexOf("."));
extension = fileName.substring(fileName.lastIndexOf(".") + 1, fileName.length());
} else {
nameWithoutExtension = fileName;
}
Pattern pattern = Pattern.compile("(.*?)-(\\d+)");
Matcher matcher = pattern.matcher(nameWithoutExtension);
if (matcher.matches()) {
nameWithoutExtension = matcher.group(1);
int fileIndex = Integer.parseInt(matcher.group(2));
index = fileIndex + 1;
}
String newName = nameWithoutExtension + "-" + index;
if (extension != null) {
newName += "." + extension;
}
File newFile = file.toPath().getParent().resolve(newName).toFile();
return getEmptyFile(newFile, index + 1);
}
return file;
}
private String makeMD5Checksum(File file) throws NoSuchAlgorithmException, IOException {
MessageDigest md = MessageDigest.getInstance("MD5");
md.update(Files.readAllBytes(file.toPath()));
byte[] digest = md.digest();
String checksum = DatatypeConverter.printHexBinary(digest);
return checksum;
}
}
......@@ -75,20 +75,26 @@ public class FileDAO {
return Arrays.asList((String[]) array.getArray());
}
public void setBusy(int nodeId, String jobId) {
public int setBusy(String vosPath, String jobId) {
String sql = "UPDATE node SET job_id = ? WHERE node_id = id_from_vos_path(?)";
String sql = "UPDATE node SET job_id = ? WHERE node_id = ?";
jdbcTemplate.update(conn -> {
int nodes = jdbcTemplate.update(conn -> {
PreparedStatement ps = conn.prepareStatement(sql);
if (jobId == null) {
ps.setNull(1, Types.VARCHAR);
} else {
ps.setString(1, jobId);
}
ps.setInt(2, nodeId);
ps.setString(2, vosPath);
return ps;
});
if (nodes > 1) {
// This should never happen thanks to database constraints
throw new IllegalStateException("More than one node at path: " + vosPath);
}
return nodes;
}
public void setOsName(int nodeId, String newName) {
......@@ -192,6 +198,73 @@ public class FileDAO {
});
}
// TODO: same problem as get archive file infos
public List<FileInfo> getBranchFileInfos(String rootVosPath, String jobId) {
String sql = "SELECT n.node_id, n.is_public, n.group_read, n.group_write, n.creator_id, n.async_trans,\n"
+ "n.content_type, n.content_encoding, n.content_length, n.content_md5,\n"
+ "n.accept_views, n.provide_views, l.location_type, n.path <> n.relative_path AS virtual_parent,\n"
+ "(SELECT user_name FROM users WHERE user_id = n.creator_id) AS username,\n"
+ "base_path, get_os_path(n.node_id) AS os_path, get_vos_path(n.node_id) AS vos_path,\n"
+ "n.type = 'container' AS is_directory, n.name, n.location_id, n.job_id\n"
+ "FROM node n\n"
+ "JOIN node p ON p.path @> n.path\n"
+ "LEFT JOIN location l ON l.location_id = n.location_id\n"
+ "LEFT JOIN storage s ON s.storage_id = l.storage_dest_id\n"
+ "WHERE (p.node_id = id_from_vos_path(?) AND n.job_id = ?)\n"
+ "ORDER BY vos_path ASC\n";
return jdbcTemplate.query(conn -> {
PreparedStatement ps = conn.prepareStatement(sql);
ps.setString(1, rootVosPath);
ps.setString(2, jobId);
return ps;
}, rs -> {
List<FileInfo> fileInfos = new ArrayList<>();
while (rs.next()) {
fileInfos.add(getFileInfo(rs));
}
return fileInfos;
});
}
public void setBranchLocationId(String rootVosPath, String jobId, int locationId) {
String cte = "SELECT n.node_id AS id\n"
+ "FROM node n\n"
+ "JOIN node p ON p.path @> n.path\n"
+ "WHERE (p.node_id = id_from_vos_path(?) AND n.job_id = ?)\n";
String update = "UPDATE node\n"
+ " SET location_id = ?\n"
+ "FROM cte\n"
+ "WHERE node_id = cte.id\n";
String sql = "WITH cte AS (\n"
+ cte
+ ")\n"
+update;
jdbcTemplate.update(conn -> {
PreparedStatement ps = conn.prepareStatement(sql);
ps.setString(1, rootVosPath);
ps.setString(2, jobId);
ps.setInt(3, locationId);
return ps;
});
}
// !! duplicate code from NodeDAO
public void releaseBusyNodesByJobId(String jobId) {
String sql = "UPDATE node SET job_id = NULL WHERE job_id = ?";
jdbcTemplate.update(conn -> {
PreparedStatement ps = conn.prepareStatement(sql);
ps.setString(1, jobId);
return ps;
});
}
private FileInfo getFileInfo(ResultSet rs) throws SQLException {
FileInfo fi = new FileInfo();
fi.setNodeId(rs.getInt("node_id"));
......@@ -213,7 +286,7 @@ public class FileDAO {
fi.setContentMd5(rs.getString("content_md5"));
fi.setContentType(rs.getString("content_type"));
fi.setDirectory(rs.getBoolean("is_directory"));
fi.setJobId(rs.getString("job_id"));
fi.setJobId(rs.getString("job_id"));
int locationId = rs.getInt("location_id");
if (!rs.wasNull()) {
fi.setLocationId(locationId);
......
......@@ -180,4 +180,8 @@ public class FileInfo {
public void setJobId(String jobId) {
this.jobId = jobId;
}
public static String getVosParentPath(FileInfo fileInfo) {
return fileInfo.getVirtualPath().substring(0, fileInfo.getVirtualPath().lastIndexOf("/"));
}
}
/*
* This file is part of vospace-file-service
* Copyright (C) 2021 Istituto Nazionale di Astrofisica
* SPDX-License-Identifier: GPL-3.0-or-later
*/
package it.inaf.ia2.transfer.service;
import it.inaf.ia2.transfer.auth.TokenPrincipal;
import it.inaf.ia2.transfer.persistence.FileDAO;
import it.inaf.ia2.transfer.persistence.LocationDAO;
import it.inaf.ia2.transfer.persistence.model.FileInfo;
import it.inaf.oats.vospace.exception.InternalFaultException;
import it.inaf.oats.vospace.exception.NodeNotFoundException;
import it.inaf.oats.vospace.exception.PermissionDeniedException;
import it.inaf.oats.vospace.exception.QuotaExceededException;
import java.io.File;
import java.io.InputStream;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.stereotype.Service;
import org.springframework.web.client.RestTemplate;
@Service
public class FileCopyService {
private static final Logger LOG = LoggerFactory.getLogger(FileCopyService.class);
@Autowired
private FileDAO fileDAO;
@Autowired
private LocationDAO locationDAO;
@Autowired
private AuthorizationService authorizationService;
@Autowired
private RestTemplate restTemplate;
@Autowired
private PutFileService putFileService;
@Value("${upload_location_id}")
private int uploadLocationId;
public void copyFiles(String sourceRootVosPath,
String destinationRootVosPath, String jobId, TokenPrincipal principal) {
LOG.trace("copyFiles called for source {}, destination {}, from jobId \"{}\"",
sourceRootVosPath, destinationRootVosPath, jobId);
// We use jobId to identify nodes created by the REST part of CopyNode
// We expect them to be locked
List<FileInfo> sources
= fileDAO.getBranchFileInfos(sourceRootVosPath, jobId);
LOG.debug("found {} sources", sources.size());
if (sources.isEmpty()) {
throw new NodeNotFoundException(sourceRootVosPath);
}
// Set location of destinations to this file service update location
// before retrieving file infos
fileDAO.setBranchLocationId(destinationRootVosPath, jobId, uploadLocationId);
List<FileInfo> destinations
= fileDAO.getBranchFileInfos(destinationRootVosPath, jobId);
LOG.debug("found {} destinations", destinations.size());
if (destinations.isEmpty()) {
throw new NodeNotFoundException(destinationRootVosPath);
}
if (sources.size() != destinations.size()) {
throw new IllegalStateException("Sources and destinations list have different sizes");
}
// Create destination directories on disk
//this.makeDirectoryStructure(destinations);
this.fillDestinations(sources,
destinations,
sourceRootVosPath,
destinationRootVosPath,
principal);
}
private void fillDestinations(List<FileInfo> sourcesFileInfos,
List<FileInfo> destinationFileInfos,
String sourceRootVosPath,
String destinationRootVosPath,
TokenPrincipal principal) {
// it will be initialized only when necessary
Map<Integer, String> portalLocationUrls = null;
for (FileInfo destinationFileInfo : destinationFileInfos) {
LOG.trace("Processing {} destination", destinationFileInfo.getVirtualPath());
// Cycle on files only
if (!destinationFileInfo.isDirectory()) {
// Calculate source file vos path
String correspondingSourceVosPath
= this.getCorrespondingSourceVosPath(sourceRootVosPath,
destinationRootVosPath,
destinationFileInfo.getVirtualPath());
// Get source fileInfo corresponding to this destination
Optional<FileInfo> sourceOpt = this.findFileInfoByVosPath(sourcesFileInfos,
correspondingSourceVosPath);
FileInfo sourceFileInfo = sourceOpt
.orElseThrow(() -> new IllegalStateException("Can't find file info for: "
+ correspondingSourceVosPath + " in source files list"));
// Get remaining quota for precheck
String parentPath = FileInfo.getVosParentPath(destinationFileInfo);
Long remainingQuota = fileDAO.getRemainingQuota(parentPath);
// Compare to source fileInfo content length
if (remainingQuota != null) {
Long sourceSize = sourceFileInfo.getContentLength();
if (sourceSize != null && remainingQuota < sourceSize) {
throw new QuotaExceededException("Path: " + destinationFileInfo.getVirtualPath());
}
}
if (sourceFileInfo.getLocationId() != null && sourceFileInfo.getLocationId() != uploadLocationId) {
// remote file
if (portalLocationUrls == null) {
portalLocationUrls = locationDAO.getPortalLocationUrls();
}
String url = portalLocationUrls.get(sourceFileInfo.getLocationId());
// download file to destination disk path
this.downloadFileToDisk(sourceFileInfo,
destinationFileInfo,
principal, url, remainingQuota);
} else {
// local file
this.copyLocalFile(sourceFileInfo, destinationFileInfo, principal, remainingQuota);
}
}
}
}
private String getCorrespondingSourceVosPath(String sourceRootVosPath,
String destinationRootVosPath,
String destinationVosPath) {
return sourceRootVosPath
+ destinationVosPath.substring(destinationRootVosPath.length());
}
private Optional<FileInfo> findFileInfoByVosPath(List<FileInfo> list, String vosPath) {
return list.stream().filter(i -> i.getVirtualPath().equals(vosPath)).findFirst();
}
private void downloadFileToDisk(FileInfo sourceFileInfo,
FileInfo destinationFileInfo, TokenPrincipal tokenPrincipal, String baseUrl, Long remainingQuota) {
if (baseUrl == null) {
LOG.error("Location URL not found for location " + sourceFileInfo.getLocationId());
throw new InternalFaultException("Unable to retrieve location of file " + sourceFileInfo.getVirtualPath());
}
String url = baseUrl + "/" + sourceFileInfo.getVirtualName();
LOG.trace("Downloading file from {}", url);
restTemplate.execute(url, HttpMethod.GET, req -> {
HttpHeaders headers = req.getHeaders();
if (tokenPrincipal.getToken() != null) {
headers.setBearerAuth(tokenPrincipal.getToken());
}
}, res -> {
try (InputStream in = res.getBody()) {
putFileService.storeFileFromInputStream(sourceFileInfo, destinationFileInfo, in, remainingQuota);
} catch (Exception ex) {
// outFile.delete();
throw new RuntimeException(ex);
}
return null;
}, new Object[]{});
}
private void copyLocalFile(FileInfo sourceFileInfo,
FileInfo destinationFileInfo, TokenPrincipal tokenPrincipal, Long remainingQuota) {
// Check permission
if (!authorizationService.isDownloadable(sourceFileInfo, tokenPrincipal)) {
throw PermissionDeniedException.forPath(sourceFileInfo.getVirtualPath());
}
File file = new File(sourceFileInfo.getOsPath());
LOG.trace("Copying file: {} to {}",file.getAbsolutePath(), destinationFileInfo.getOsPath());
putFileService.copyLocalFile(sourceFileInfo, destinationFileInfo, remainingQuota);
}
}
/*
* This file is part of vospace-file-service
* Copyright (C) 2021 Istituto Nazionale di Astrofisica
* SPDX-License-Identifier: GPL-3.0-or-later
*/
package it.inaf.ia2.transfer.service;
import it.inaf.ia2.transfer.persistence.FileDAO;
import it.inaf.ia2.transfer.persistence.model.FileInfo;
import it.inaf.oats.vospace.exception.QuotaExceededException;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.UncheckedIOException;
import java.nio.file.Files;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.Objects;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.xml.bind.DatatypeConverter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@Service
public class PutFileService {
private static final Logger LOG = LoggerFactory.getLogger(PutFileService.class);
@Autowired
private FileDAO fileDAO;
public synchronized void makeFoldersPath(File file) {
/**
* This method must be synchronized, to avoid concurrency issues when
* multiple files are uploaded to a new folder in parallel.
*/
if (!file.getParentFile().exists()) {
if (!file.getParentFile().mkdirs()) {
throw new IllegalStateException("Unable to create parent folder: " + file.getParentFile().getAbsolutePath());
}
}
}
public void copyLocalFile(FileInfo sourceFileInfo, FileInfo destinationFileInfo, Long remainingQuota) {
File destinationFile = this.prepareDestination(destinationFileInfo);
File sourceFile = new File(sourceFileInfo.getOsPath());
try {
Files.copy(sourceFile.toPath(), destinationFile.toPath());
this.finalizeFile(sourceFileInfo, destinationFileInfo, destinationFile, remainingQuota);
} catch (IOException e) {
destinationFile.delete();
throw new UncheckedIOException(e);
} catch (NoSuchAlgorithmException e) {
throw new RuntimeException(e);
}
}
public void storeFileFromInputStream(FileInfo destinationFileInfo,
InputStream is, Long remainingQuota) {
this.storeFileFromInputStream(null, destinationFileInfo, is, remainingQuota);
}
public void storeFileFromInputStream(FileInfo sourceFileInfo, FileInfo destinationFileInfo,
InputStream is, Long remainingQuota) {
File destinationFile = this.prepareDestination(destinationFileInfo);
try {
Files.copy(is, destinationFile.toPath());
this.finalizeFile(sourceFileInfo, destinationFileInfo, destinationFile, remainingQuota);
} catch (IOException e) {
destinationFile.delete();
throw new UncheckedIOException(e);
} catch (NoSuchAlgorithmException e) {
destinationFile.delete();
throw new RuntimeException(e);
}
}
private File prepareDestination(FileInfo destinationFileInfo) {
File file = new File(destinationFileInfo.getOsPath());
makeFoldersPath(file);
String originalFileName = file.getName();
file = getEmptyFile(file, 1);
if (!originalFileName.equals(file.getName())) {
fileDAO.setOsName(destinationFileInfo.getNodeId(), file.getName());
}
return file;
}
private Long checkQuota(File destinationFile, Long remainingQuota) throws IOException {
Long fileSize = Files.size(destinationFile.toPath());
// Quota limit is checked again to handle cases where MultipartFile is not used
if (remainingQuota != null && fileSize > remainingQuota) {
destinationFile.delete();
throw new QuotaExceededException("Path: " + destinationFile.toPath().toString());
}
return fileSize;
}
private void finalizeFile(FileInfo sourceFileInfo, FileInfo destinationFileInfo,
File destinationFile, Long remainingQuota) throws IOException, NoSuchAlgorithmException {
Long fileSize = this.checkQuota(destinationFile, remainingQuota);
if (destinationFileInfo.getContentType() == null) {
destinationFileInfo.setContentType(Files.probeContentType(destinationFile.toPath()));
}
String md5Checksum = makeMD5Checksum(destinationFile);
// TODO: discuss if mismatches lead to taking actions
if (sourceFileInfo != null) {
if (!Objects.equals(sourceFileInfo.getContentLength(), fileSize)) {
LOG.warn("Destination file {} size mismatch with source", destinationFile.toPath().toString());
}
if (sourceFileInfo.getContentType() != null &&
!sourceFileInfo.getContentType().equals(destinationFileInfo.getContentType())) {
LOG.warn("Destination file {} content type mismatch with source {} {}", destinationFile.toPath().toString(),
destinationFileInfo.getContentType(), sourceFileInfo.getContentType());
}
if (sourceFileInfo.getContentMd5() != null &&
!sourceFileInfo.getContentMd5().equals(md5Checksum)) {
LOG.warn("Destination file {} md5 mismatch with source {} {}", destinationFile.toPath().toString(),
destinationFileInfo.getContentMd5(), sourceFileInfo.getContentMd5() );
}
}
fileDAO.updateFileAttributes(destinationFileInfo.getNodeId(),
destinationFileInfo.getContentType(),
destinationFileInfo.getContentEncoding(),
fileSize,
md5Checksum);
}
/**
* Handles duplicate file uploads generating a new non existent path. This
* is necessary in some edge cases, like when a file has been renamed in
* VOSpace only but the original file on disk still has the old name or if a
* file has been marked for deletion and a file with the same name is
* uploaded before the cleanup.
*/
private File getEmptyFile(File file, int index) {
if (file.exists()) {
String fileName = file.getName();
String nameWithoutExtension;
String extension = null;
if (fileName.contains(".")) {
nameWithoutExtension = fileName.substring(0, fileName.lastIndexOf("."));
extension = fileName.substring(fileName.lastIndexOf(".") + 1, fileName.length());
} else {
nameWithoutExtension = fileName;
}
Pattern pattern = Pattern.compile("(.*?)-(\\d+)");
Matcher matcher = pattern.matcher(nameWithoutExtension);
if (matcher.matches()) {
nameWithoutExtension = matcher.group(1);
int fileIndex = Integer.parseInt(matcher.group(2));
index = fileIndex + 1;
}
String newName = nameWithoutExtension + "-" + index;
if (extension != null) {
newName += "." + extension;
}
File newFile = file.toPath().getParent().resolve(newName).toFile();
return getEmptyFile(newFile, index + 1);
}
return file;
}
private String makeMD5Checksum(File file) throws NoSuchAlgorithmException, IOException {
MessageDigest md = MessageDigest.getInstance("MD5");
md.update(Files.readAllBytes(file.toPath()));
byte[] digest = md.digest();
String checksum = DatatypeConverter.printHexBinary(digest);
return checksum;
}
}
/*
* This file is part of vospace-file-service
* Copyright (C) 2021 Istituto Nazionale di Astrofisica
* SPDX-License-Identifier: GPL-3.0-or-later
*/
package it.inaf.ia2.transfer.controller;
import com.fasterxml.jackson.databind.ObjectMapper;
import it.inaf.ia2.transfer.auth.TokenPrincipal;
import it.inaf.ia2.transfer.persistence.JobDAO;
import it.inaf.ia2.transfer.service.FileCopyService;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.http.MediaType;
import org.springframework.test.web.servlet.MockMvc;
import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post;
import static org.springframework.test.web.servlet.result.MockMvcResultHandlers.print;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
@SpringBootTest
@AutoConfigureMockMvc(addFilters = false)
public class CopyControllerTest {
@MockBean
private JobDAO jobDao;
@Autowired
private MockMvc mockMvc;
@MockBean
private FileCopyService fileCopyService;
private static String jobId;
private static String sourceVosRootPath;
private static String destVosRootPath;
private static String filename;
private static final ObjectMapper MAPPER = new ObjectMapper();
@BeforeAll
public static void setUp() {
jobId = "pippo_copy";
filename = "file.txt";
sourceVosRootPath = "/source/" + filename;
destVosRootPath = "/destination/" + filename;
}
@Test
public void testCopyControllerNoJobMatching() throws Exception {
CopyRequest copyRequest = this.getCopyRequest(jobId,
sourceVosRootPath, destVosRootPath);
when(jobDao.isJobExisting(eq(jobId))).thenReturn(false);
mockMvc.perform(post("/copy")
.principal(fakePrincipal("user_copy"))
.contentType(MediaType.APPLICATION_JSON)
.content(MAPPER.writeValueAsString(copyRequest)))
.andDo(print())
.andExpect(status().is4xxClientError());
}
@Test
public void testCopyControllerAnonymousForbidden() throws Exception {
CopyRequest copyRequest = this.getCopyRequest(jobId,
sourceVosRootPath, destVosRootPath);
when(jobDao.isJobExisting(eq(jobId))).thenReturn(true);
mockMvc.perform(post("/copy")
.principal(fakePrincipal("anonymous"))
.contentType(MediaType.APPLICATION_JSON)
.content(MAPPER.writeValueAsString(copyRequest)))
.andDo(print())
.andExpect(status().is4xxClientError());
}
private CopyRequest getCopyRequest(String jobId,
String sourceRootVosPath, String destinationRootVosPath) {
CopyRequest result = new CopyRequest();
result.setJobId(jobId);
result.setSourceRootVosPath(sourceRootVosPath);
result.setDestinationRootVosPath(destinationRootVosPath);
return result;
}
private TokenPrincipal fakePrincipal(String name) {
TokenPrincipal principal = mock(TokenPrincipal.class);
when(principal.getName()).thenReturn(name);
return principal;
}
}
......@@ -94,7 +94,7 @@ public class PutFileControllerTest {
assertEquals("content", Files.contentOf(file, StandardCharsets.UTF_8));
assertTrue(file.delete());
}
@Test
public void putGenericFileWithNameConflictExtension() throws Exception {
putGenericFileWithNameConflict("test.txt", "test-1.txt", "test-2.txt");
......@@ -299,6 +299,7 @@ public class PutFileControllerTest {
fileInfo.setPublic(false);
when(fileDao.getFileInfo(any())).thenReturn(Optional.of(fileInfo));
when(fileDao.setBusy(any(), any())).thenReturn(1);
return fileInfo;
}
......
......@@ -42,6 +42,26 @@ public class FileDAOTest {
ReflectionTestUtils.setField(dao, "uploadLocationId", uploadLocationId);
}
@Test
public void testGetBranchFileInfo() {
List<FileInfo> fi = dao.getBranchFileInfos("/test100", "pippo");
assertEquals(3, fi.size());
List<FileInfo> fi2 = dao.getBranchFileInfos("/test100/test1001.txt", "pippo");
assertEquals(1, fi2.size());
}
@Test
public void testSetBranchLocationId() {
dao.setBranchLocationId("/test100", "pippo", 3);
List<FileInfo> fi = dao.getBranchFileInfos("/test100", "pippo");
assertEquals(3, fi.size());
fi.forEach(f -> {
assertEquals(3, f.getLocationId());
});
}
@Test
public void testGetFileInfo() {
......@@ -93,13 +113,26 @@ public class FileDAOTest {
FileInfo fileInfo = dao.getFileInfo("/public/file1").get();
assertNull(fileInfo.getJobId());
dao.setBusy(fileInfo.getNodeId(), "pippo1");
dao.setBusy(fileInfo.getVirtualPath(), "pippo1");
assertEquals("pippo1", dao.getFileInfo("/public/file1").get().getJobId());
dao.setBusy(fileInfo.getNodeId(), null);
dao.setBusy(fileInfo.getVirtualPath(), null);
assertNull(dao.getFileInfo("/public/file1").get().getJobId());
}
@Test
public void testReleaseNodesByJobId() {
List<FileInfo> fi = dao.getBranchFileInfos("/test100", "pippo");
assertEquals(3, fi.size());
dao.releaseBusyNodesByJobId("pippo");
fi = dao.getBranchFileInfos("/test100", "pippo");
assertEquals(0, fi.size());
}
@Test
public void testSetOsName() {
......
/*
* This file is part of vospace-file-service
* Copyright (C) 2021 Istituto Nazionale di Astrofisica
* SPDX-License-Identifier: GPL-3.0-or-later
*/
package it.inaf.ia2.transfer.service;
import it.inaf.ia2.transfer.auth.TokenPrincipal;
import it.inaf.ia2.transfer.persistence.model.FileInfo;
import it.inaf.ia2.transfer.persistence.FileDAO;
import it.inaf.ia2.transfer.persistence.JobDAO;
import java.io.File;
import java.util.List;
import org.apache.commons.io.FileUtils;
import org.assertj.core.util.Files;
import org.junit.jupiter.api.AfterEach;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.test.web.servlet.MockMvc;
import org.springframework.util.FileSystemUtils;
@SpringBootTest
@AutoConfigureMockMvc(addFilters = false)
public class CopyServiceTest {
@MockBean
private FileDAO fileDao;
@MockBean
private JobDAO jobDao;
@Autowired
private MockMvc mockMvc;
@Autowired
private FileCopyService fileCopyService;
private static File sourceDirectory;
private static File destinationDirectory;
private static File sourceFile;
private static String jobId;
private static String sourceVosRootPath;
private static String destVosRootPath;
private static String filename;
private static String userId;
@BeforeAll
public static void setUp() {
jobId = "pippo_copy";
userId = "user_copy";
filename = "file.txt";
sourceVosRootPath = "/source/" + filename;
destVosRootPath = "/destination/" + filename;
}
@BeforeEach
public void setUpPerTest() throws Exception {
sourceDirectory = Files.newTemporaryFolder();
destinationDirectory = Files.newTemporaryFolder();
sourceFile = this.createFile(sourceDirectory, filename);
}
@AfterEach
public void tearDown() {
FileSystemUtils.deleteRecursively(sourceDirectory);
FileSystemUtils.deleteRecursively(destinationDirectory);
}
@Test
public void testCopyService() throws Exception {
when(fileDao.getBranchFileInfos(eq(sourceVosRootPath), eq(jobId)))
.thenReturn(
List.of(this.createBaseFileInfo(
filename, sourceDirectory, sourceVosRootPath)));
when(fileDao.getBranchFileInfos(eq(destVosRootPath), eq(jobId)))
.thenReturn(
List.of(this.createBaseFileInfo(
filename, destinationDirectory, destVosRootPath)));
when(fileDao.getRemainingQuota(any())).thenReturn(null);
assertTrue(sourceFile.exists());
File destinationFile = destinationDirectory.toPath().resolve(filename).toFile();
assertFalse(destinationFile.exists());
fileCopyService.copyFiles(
sourceVosRootPath, destVosRootPath, jobId, fakePrincipal(userId));
assertTrue(sourceFile.exists());
assertTrue(destinationFile.exists());
assertTrue(FileUtils.contentEquals(sourceFile, destinationFile));
}
private FileInfo createBaseFileInfo(String fileName, File directory, String vosPath) {
FileInfo fileInfo = new FileInfo();
fileInfo.setOsPath(getTestFilePath(fileName, directory));
fileInfo.setVirtualPath(vosPath);
fileInfo.setPublic(false);
fileInfo.setOwnerId(userId);
return fileInfo;
}
private String getTestFilePath(String fileName, File directory) {
return directory.toPath().resolve(fileName).toFile().getAbsolutePath();
}
private File createFile(File parent, String filename) throws Exception {
parent.mkdir();
File file = parent.toPath().resolve(filename).toFile();
file.createNewFile();
java.nio.file.Files.write(file.toPath(), "content_of_file".getBytes());
return file;
}
private TokenPrincipal fakePrincipal(String name) {
TokenPrincipal principal = mock(TokenPrincipal.class);
when(principal.getName()).thenReturn(name);
when(principal.getToken()).thenReturn("any_token_not_null");
return principal;
}
}
......@@ -37,6 +37,12 @@ INSERT INTO node (parent_path, parent_relative_path, name, type, creator_id, con
('12.13', NULL, 'file1', 'data', 'user1', 100000, 500000),
('12.13', NULL, 'file2', 'data', 'user1', 200000, 500000);
-- test data for get branch file info
INSERT INTO node (parent_path, parent_relative_path, name, type, creator_id, group_read, group_write, job_id) VALUES ('', NULL, 'test100', 'container', 'user1', '{"group1","group2"}','{"group2"}', 'pippo'); -- /test100
INSERT INTO node (parent_path, parent_relative_path, name, type, creator_id, group_read, group_write, job_id) VALUES ('16', '', 'test1001.txt', 'data', 'user1', '{"group1","group2"}','{"group2"}', 'pippo'); -- /test100
INSERT INTO node (parent_path, parent_relative_path, name, type, creator_id, group_read, group_write, job_id) VALUES ('16', '', 'test1002.txt', 'data', 'user1', '{"group1","group2"}','{"group2"}', 'pippo'); -- /test100
INSERT INTO node (parent_path, parent_relative_path, name, type, creator_id, group_read, group_write, job_id) VALUES ('16', '', 'test1003.txt', 'data', 'user1', '{"group1","group2"}','{"group2"}', NULL); -- /test100
DELETE FROM job;
INSERT INTO job (job_id, owner_id, job_type, phase, start_time, end_time, creation_time, job_info, results) VALUES ('pippo1', 'user1', 'pullFromVoSpace', 'ARCHIVED', NULL, NULL, '2011-06-22 19:10:25', NULL, NULL);
......