Commit 4fc03d51 authored by Nicola Fulvio Calabria's avatar Nicola Fulvio Calabria
Browse files

Restructured code to have a common service for disk writes

parent 3388d413

src/main/java/it/inaf/ia2/transfer/controller/AuthenticatedFileController.java

+1 −1
Original line number Diff line number Diff line
@@ -14,7 +14,7 @@ public abstract class AuthenticatedFileController extends FileController { 
        TokenPrincipal principal = (TokenPrincipal) request.getUserPrincipal();



        if ("anonymous".equals(principal.getName())) {

            throw new PermissionDeniedException("Tar/Zip archive generation not allowed to anonymous users");

            throw new PermissionDeniedException(this.getCustomAuthErrorMessage());

        }



        return principal;

src/main/java/it/inaf/ia2/transfer/controller/CopyController.java

+34 −8
Original line number Diff line number Diff line
@@ -5,6 +5,7 @@ 
 */

package it.inaf.ia2.transfer.controller;



import it.inaf.ia2.transfer.persistence.FileDAO;

import org.springframework.http.MediaType;

import org.springframework.http.ResponseEntity;

import org.springframework.web.bind.annotation.PostMapping;
@@ -12,29 +13,54 @@ import org.springframework.web.bind.annotation.RequestBody; 
import org.springframework.web.bind.annotation.RestController;

import org.springframework.beans.factory.annotation.Autowired;

import it.inaf.ia2.transfer.service.FileCopyService;

import it.inaf.oats.vospace.exception.InvalidArgumentException;

import java.util.concurrent.CompletableFuture;

import org.slf4j.Logger;

import org.slf4j.LoggerFactory;



@RestController

public class CopyController extends AuthenticatedFileController {

    

    private static final Logger LOG = LoggerFactory.getLogger(CopyController.class);



    @Autowired

    private FileCopyService copyService;

    

    @Autowired 

    private FileDAO fileDao;



    @PostMapping(value = "/copy", consumes = MediaType.APPLICATION_JSON_VALUE)

    public ResponseEntity<?> copyFiles(@RequestBody CopyRequest copyRequest) {



        String jobId = copyRequest.getJobId();



        if (jobId == null) {

            throw new InvalidArgumentException("Job Id cannot be null");

        } else if (!jobDAO.isJobExisting(jobId)) {

            throw new InvalidArgumentException("Job " + jobId + " not found");

        }

        

        LOG.debug("copyFiles called from jobId {}", jobId);



        // need to make a completable future start

        CompletableFuture.runAsync(() -> {

            handleFileJob(() -> copyService.copyFiles(copyRequest.getSourceRootVosPath(), 

            handleFileJob(() -> {

                    try {

                    copyService.copyFiles(copyRequest.getSourceRootVosPath(),

                    copyRequest.getDestinationRootVosPath(), copyRequest.getJobId(),

                getPrincipal()), copyRequest.jobId);

                    getPrincipal());

                    } finally {

                        // TODO: cleanup code to remove unpopulated nodes in case

                        // of failure?

                        fileDao.releaseBusyNodesByJobId(jobId);

                    }

            }, jobId);

        });



        return ResponseEntity.ok(

                copyRequest.getJobId() + " copy task accepted by File Service"

        );



        

    }



    @Override

src/main/java/it/inaf/ia2/transfer/controller/CopyRequest.java

+3 −3
Original line number Diff line number Diff line
@@ -7,9 +7,9 @@ package it.inaf.ia2.transfer.controller; 


public class CopyRequest {



    String jobId;

    String sourceRootVosPath;

    String destinationRootVosPath;

    private String jobId;

    private String sourceRootVosPath;

    private String destinationRootVosPath;



    public String getJobId() {

        return jobId;

src/main/java/it/inaf/ia2/transfer/controller/FileController.java

+1 −1
Original line number Diff line number Diff line
@@ -26,7 +26,7 @@ public abstract class FileController { 
    protected HttpServletRequest request;



    @Autowired

    private JobDAO jobDAO;

    protected JobDAO jobDAO;



    public String getPath() {

src/main/java/it/inaf/ia2/transfer/controller/PutFileController.java

+36 −122
Original line number Diff line number Diff line
@@ -8,6 +8,7 @@ package it.inaf.ia2.transfer.controller; 
import it.inaf.ia2.transfer.persistence.model.FileInfo;

import it.inaf.ia2.transfer.persistence.FileDAO;

import it.inaf.ia2.transfer.persistence.JobDAO;

import it.inaf.ia2.transfer.service.PutFileService;

import it.inaf.oats.vospace.exception.InvalidArgumentException;

import it.inaf.oats.vospace.exception.NodeNotFoundException;

import it.inaf.oats.vospace.exception.QuotaExceededException;
@@ -41,7 +42,7 @@ public class PutFileController extends FileController { 
    private FileDAO fileDAO;



    @Autowired

    private JobDAO jobDAO;

    private PutFileService putFileService;



    @PutMapping("/**")

    public void putFile(@RequestHeader(value = HttpHeaders.CONTENT_ENCODING, required = false) String contentEncoding,
@@ -62,12 +63,20 @@ public class PutFileController extends FileController { 
        }



        handleFileJob(() -> {

            Optional<FileInfo> optFileInfo = fileDAO.getFileInfo(path);



            if (optFileInfo.isPresent()) {

                FileInfo fileInfo = optFileInfo.get();

            int nodes = fileDAO.setBusy(path, jobId);



                String parentPath = fileInfo.getVirtualPath().substring(0, fileInfo.getVirtualPath().lastIndexOf("/"));

            if (nodes == 1) {

                try {

                    FileInfo fileInfo

                            = fileDAO.getFileInfo(path).orElseThrow(

                                    // This can happen only if some code ignores busy state

                                    // and deletes the node

                                    () -> {

                                        throw new NodeNotFoundException(path);

                                    });



                    String parentPath = FileInfo.getVosParentPath(fileInfo);

                    Long remainingQuota = fileDAO.getRemainingQuota(parentPath);



                    // if MultipartFile provides file size it is possible to check
@@ -82,112 +91,17 @@ public class PutFileController extends FileController { 
                    fileInfo.setContentEncoding(contentEncoding);



                    try (InputStream in = file != null ? file.getInputStream() : request.getInputStream()) {

                    storeGenericFile(fileInfo, in, jobId, remainingQuota);

                } catch (IOException | NoSuchAlgorithmException ex) {

                        putFileService.storeFileFromInputStream(fileInfo, in, remainingQuota);

                    } catch (Exception ex) {

                        throw new RuntimeException(ex);

                    }

            } else {

                throw new NodeNotFoundException(path);

            }

        }, jobId);

    }



    private void storeGenericFile(FileInfo fileInfo, InputStream is, String jobId, Long remainingQuota) throws IOException, NoSuchAlgorithmException {



        File file = new File(fileInfo.getOsPath());



        /**

         * This block must be synchronized, to avoid concurrency issues when

         * multiple files are uploaded to a new folder in parallel.

         */

        synchronized (this) {

            if (!file.getParentFile().exists()) {

                if (!file.getParentFile().mkdirs()) {

                    throw new IllegalStateException("Unable to create parent folder: " + file.getParentFile().getAbsolutePath());

                }

            }

        }



        String originalFileName = file.getName();

        file = getEmptyFile(file, 1);

        if (!originalFileName.equals(file.getName())) {

            fileDAO.setOsName(fileInfo.getNodeId(), file.getName());

        }



        try {

            fileDAO.setBusy(fileInfo.getNodeId(), jobId);

            Files.copy(is, file.toPath());



            if (fileInfo.getContentType() == null) {

                fileInfo.setContentType(Files.probeContentType(file.toPath()));

            }



            Long fileSize = Files.size(file.toPath());



            // Quota limit is checked again to handle cases where MultipartFile is not used

            if (remainingQuota != null && fileSize > remainingQuota) {

                file.delete();

                throw new QuotaExceededException("Path: " + fileInfo.getVirtualPath());

            }



            String md5Checksum = makeMD5Checksum(file);



            fileDAO.updateFileAttributes(fileInfo.getNodeId(),

                    fileInfo.getContentType(),

                    fileInfo.getContentEncoding(),

                    fileSize,

                    md5Checksum);

                } finally {

            fileDAO.setBusy(fileInfo.getNodeId(), null);

        }

                    fileDAO.setBusy(path, null);

                }



    /**

     * Handles duplicate file uploads generating a new non existent path. This

     * is necessary in some edge cases, like when a file has been renamed in

     * VOSpace only but the original file on disk still has the old name or if a

     * file has been marked for deletion and a file with the same name is

     * uploaded before the cleanup.

     */

    private File getEmptyFile(File file, int index) {

        if (file.exists()) {



            String fileName = file.getName();



            String nameWithoutExtension;

            String extension = null;

            if (fileName.contains(".")) {

                nameWithoutExtension = fileName.substring(0, fileName.lastIndexOf("."));

                extension = fileName.substring(fileName.lastIndexOf(".") + 1, fileName.length());

            } else {

                nameWithoutExtension = fileName;

            }



            Pattern pattern = Pattern.compile("(.*?)-(\\d+)");

            Matcher matcher = pattern.matcher(nameWithoutExtension);

            if (matcher.matches()) {

                nameWithoutExtension = matcher.group(1);

                int fileIndex = Integer.parseInt(matcher.group(2));

                index = fileIndex + 1;

            }



            String newName = nameWithoutExtension + "-" + index;

            if (extension != null) {

                newName += "." + extension;

            }



            File newFile = file.toPath().getParent().resolve(newName).toFile();

            return getEmptyFile(newFile, index + 1);

        }

        return file;

                throw new NodeNotFoundException(path);

            }



    private String makeMD5Checksum(File file) throws NoSuchAlgorithmException, IOException {

        MessageDigest md = MessageDigest.getInstance("MD5");

        md.update(Files.readAllBytes(file.toPath()));

        byte[] digest = md.digest();

        String checksum = DatatypeConverter.printHexBinary(digest);

        return checksum;

        }, jobId);

    }



}