/* * This file is part of vospace-file-service * Copyright (C) 2021 Istituto Nazionale di Astrofisica * SPDX-License-Identifier: GPL-3.0-or-later */ package it.inaf.ia2.transfer.service; import it.inaf.ia2.transfer.auth.TokenPrincipal; import it.inaf.ia2.transfer.persistence.FileDAO; import it.inaf.ia2.transfer.persistence.LocationDAO; import it.inaf.ia2.transfer.persistence.model.FileInfo; import it.inaf.oats.vospace.exception.InternalFaultException; import it.inaf.oats.vospace.exception.NodeNotFoundException; import it.inaf.oats.vospace.exception.PermissionDeniedException; import it.inaf.oats.vospace.exception.QuotaExceededException; import java.io.File; import java.io.InputStream; import java.util.List; import java.util.Map; import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpMethod; import org.springframework.stereotype.Service; import org.springframework.web.client.RestTemplate; @Service public class FileCopyService { private static final Logger LOG = LoggerFactory.getLogger(FileCopyService.class); @Autowired private FileDAO fileDAO; @Autowired private LocationDAO locationDAO; @Autowired private AuthorizationService authorizationService; @Autowired private RestTemplate restTemplate; @Autowired private PutFileService putFileService; @Value("${upload_location_id}") private int uploadLocationId; public void copyFiles(String sourceRootVosPath, String destinationRootVosPath, String jobId, TokenPrincipal principal) { LOG.trace("copyFiles called for source {}, destination {}, from jobId \"{}\"", sourceRootVosPath, destinationRootVosPath, jobId); // We use jobId to identify nodes created by the REST part of CopyNode // We expect them to be locked List sources = fileDAO.getBranchFileInfos(sourceRootVosPath, jobId); LOG.debug("found {} sources", sources.size()); if (sources.isEmpty()) { throw new NodeNotFoundException(sourceRootVosPath); } // Set location of destinations to this file service update location // before retrieving file infos fileDAO.setBranchLocationId(destinationRootVosPath, jobId, uploadLocationId); List destinations = fileDAO.getBranchFileInfos(destinationRootVosPath, jobId); LOG.debug("found {} destinations", destinations.size()); if (destinations.isEmpty()) { throw new NodeNotFoundException(destinationRootVosPath); } if (sources.size() != destinations.size()) { throw new IllegalStateException("Sources and destinations list have different sizes"); } // Create destination directories on disk //this.makeDirectoryStructure(destinations); this.fillDestinations(sources, destinations, sourceRootVosPath, destinationRootVosPath, principal); } private void fillDestinations(List sourcesFileInfos, List destinationFileInfos, String sourceRootVosPath, String destinationRootVosPath, TokenPrincipal principal) { // it will be initialized only when necessary Map portalLocationUrls = null; for (FileInfo destinationFileInfo : destinationFileInfos) { LOG.trace("Processing {} destination", destinationFileInfo.getVirtualPath()); // Cycle on files only if (!destinationFileInfo.isDirectory() && !destinationFileInfo.isLink()) { // Calculate source file vos path String correspondingSourceVosPath = this.getCorrespondingSourceVosPath(sourceRootVosPath, destinationRootVosPath, destinationFileInfo.getVirtualPath()); // Get source fileInfo corresponding to this destination Optional sourceOpt = this.findFileInfoByVosPath(sourcesFileInfos, correspondingSourceVosPath); FileInfo sourceFileInfo = sourceOpt .orElseThrow(() -> new IllegalStateException("Can't find file info for: " + correspondingSourceVosPath + " in source files list")); // Get remaining quota for precheck String parentPath = FileInfo.getVosParentPath(destinationFileInfo); Long remainingQuota = fileDAO.getRemainingQuota(parentPath); // Compare to source fileInfo content length if (remainingQuota != null) { Long sourceSize = sourceFileInfo.getContentLength(); if (sourceSize != null && remainingQuota < sourceSize) { throw new QuotaExceededException("Path: " + destinationFileInfo.getVirtualPath()); } } if (sourceFileInfo.getLocationId() != null && sourceFileInfo.getLocationId() != uploadLocationId) { // remote file if (portalLocationUrls == null) { portalLocationUrls = locationDAO.getPortalLocationUrls(); } String url = portalLocationUrls.get(sourceFileInfo.getLocationId()); // download file to destination disk path this.downloadFileToDisk(sourceFileInfo, destinationFileInfo, principal, url, remainingQuota); } else { // local file this.copyLocalFile(sourceFileInfo, destinationFileInfo, principal, remainingQuota); } } } } private String getCorrespondingSourceVosPath(String sourceRootVosPath, String destinationRootVosPath, String destinationVosPath) { return sourceRootVosPath + destinationVosPath.substring(destinationRootVosPath.length()); } private Optional findFileInfoByVosPath(List list, String vosPath) { return list.stream().filter(i -> i.getVirtualPath().equals(vosPath)).findFirst(); } private void downloadFileToDisk(FileInfo sourceFileInfo, FileInfo destinationFileInfo, TokenPrincipal tokenPrincipal, String baseUrl, Long remainingQuota) { if (baseUrl == null) { LOG.error("Location URL not found for location " + sourceFileInfo.getLocationId()); throw new InternalFaultException("Unable to retrieve location of file " + sourceFileInfo.getVirtualPath()); } String url = baseUrl + "/" + sourceFileInfo.getVirtualName(); LOG.trace("Downloading file from {}", url); restTemplate.execute(url, HttpMethod.GET, req -> { HttpHeaders headers = req.getHeaders(); if (tokenPrincipal.getToken() != null) { headers.setBearerAuth(tokenPrincipal.getToken()); } }, res -> { try (InputStream in = res.getBody()) { putFileService.storeFileFromInputStream(sourceFileInfo, destinationFileInfo, in, remainingQuota); } catch (Exception ex) { // outFile.delete(); throw new RuntimeException(ex); } return null; }, new Object[]{}); } private void copyLocalFile(FileInfo sourceFileInfo, FileInfo destinationFileInfo, TokenPrincipal tokenPrincipal, Long remainingQuota) { // Check permission if (!authorizationService.isDownloadable(sourceFileInfo, tokenPrincipal)) { throw PermissionDeniedException.forPath(sourceFileInfo.getVirtualPath()); } File file = new File(sourceFileInfo.getOsPath()); LOG.trace("Copying file: {} to {}",file.getAbsolutePath(), destinationFileInfo.getOsPath()); putFileService.copyLocalFile(sourceFileInfo, destinationFileInfo, remainingQuota); } }