Skip to content
GitLab
Explore
Sign in
VOSpace INAF
vospace-file-service
Compare revisions
62d0b750c35f26e6e4c05d79c300afc8b49771af to 209bdccfdf6cf03524a712b91cc0d7a956747e60
Commits on Source (2)
Added some Link Management features
· e6b31480
Nicola Fulvio Calabria
authored
Aug 25, 2021
e6b31480
Added location type to FileInfo and fixed issue with generation of archives containing async nodes
· 209bdccf
Sonia Zorba
authored
Sep 10, 2021
209bdccf
Hide whitespace changes
Inline
Side-by-side
src/main/java/it/inaf/ia2/transfer/persistence/FileDAO.java
View file @
209bdccf
...
...
@@ -46,7 +46,8 @@ public class FileDAO {
+
"content_type, content_encoding, content_length, content_md5, name, n.location_id,\n"
+
"accept_views, provide_views, l.location_type, n.path <> n.relative_path AS virtual_parent,\n"
+
"(SELECT user_name FROM users WHERE user_id = creator_id) AS username, n.job_id,\n"
+
"base_path, get_os_path(n.node_id) AS os_path, ? AS vos_path, false AS is_directory\n"
+
"base_path, get_os_path(n.node_id) AS os_path, ? AS vos_path, false AS is_directory,\n"
+
"type = 'link' AS is_link\n"
+
"FROM node n\n"
+
"JOIN location l ON (n.location_id IS NOT NULL AND n.location_id = l.location_id) OR (n.location_id IS NULL AND l.location_id = ?)\n"
+
"LEFT JOIN storage s ON s.storage_id = l.storage_dest_id\n"
...
...
@@ -93,7 +94,7 @@ public class FileDAO {
// This should never happen thanks to database constraints
throw
new
IllegalStateException
(
"More than one node at path: "
+
vosPath
);
}
return
nodes
;
}
...
...
@@ -174,7 +175,8 @@ public class FileDAO {
+
"n.accept_views, n.provide_views, l.location_type, n.path <> n.relative_path AS virtual_parent,\n"
+
"(SELECT user_name FROM users WHERE user_id = n.creator_id) AS username,\n"
+
"base_path, get_os_path(n.node_id) AS os_path, get_vos_path(n.node_id) AS vos_path,\n"
+
"n.type = 'container' AS is_directory, n.name, n.location_id, n.job_id\n"
+
"n.type = 'container' AS is_directory, n.name, n.location_id, n.job_id,\n"
+
"n.type = 'link' AS is_link, l.location_type\n"
+
"FROM node n\n"
+
"JOIN node p ON p.path @> n.path\n"
+
"LEFT JOIN location l ON l.location_id = n.location_id\n"
...
...
@@ -198,15 +200,16 @@ public class FileDAO {
});
}
// TODO: same problem as get archive file infos
// TODO: same problem as get archive file infos
public
List
<
FileInfo
>
getBranchFileInfos
(
String
rootVosPath
,
String
jobId
)
{
String
sql
=
"SELECT n.node_id, n.is_public, n.group_read, n.group_write, n.creator_id, n.async_trans,\n"
+
"n.content_type, n.content_encoding, n.content_length, n.content_md5,\n"
+
"n.accept_views, n.provide_views, l.location_type, n.path <> n.relative_path AS virtual_parent,\n"
+
"(SELECT user_name FROM users WHERE user_id = n.creator_id) AS username,\n"
+
"base_path, get_os_path(n.node_id) AS os_path, get_vos_path(n.node_id) AS vos_path,\n"
+
"n.type = 'container' AS is_directory, n.name, n.location_id, n.job_id\n"
+
"n.type = 'container' AS is_directory, n.name, n.location_id, n.job_id,\n"
+
"n.type = 'link' AS is_link, l.location_type\n"
+
"FROM node n\n"
+
"JOIN node p ON p.path @> n.path\n"
+
"LEFT JOIN location l ON l.location_id = n.location_id\n"
...
...
@@ -229,17 +232,17 @@ public class FileDAO {
}
public
void
setBranchLocationId
(
String
rootVosPath
,
String
jobId
,
int
locationId
)
{
String
cte
=
"SELECT n.node_id AS id\n"
+
"FROM node n\n"
+
"JOIN node p ON p.path @> n.path\n"
+
"WHERE (p.node_id = id_from_vos_path(?) AND n.job_id = ?)\n"
;
String
update
=
"UPDATE node\n"
+
" SET location_id = ?\n"
+
"FROM cte\n"
+
"WHERE node_id = cte.id\n"
;
String
sql
=
"WITH cte AS (\n"
+
cte
+
")\n"
...
...
@@ -253,16 +256,16 @@ public class FileDAO {
return
ps
;
});
}
// !! duplicate code from NodeDAO
public
void
releaseBusyNodesByJobId
(
String
jobId
)
{
String
sql
=
"UPDATE node SET job_id = NULL WHERE job_id = ?"
;
jdbcTemplate
.
update
(
conn
->
{
PreparedStatement
ps
=
conn
.
prepareStatement
(
sql
);
ps
.
setString
(
1
,
jobId
);
ps
.
setString
(
1
,
jobId
);
return
ps
;
});
});
}
private
FileInfo
getFileInfo
(
ResultSet
rs
)
throws
SQLException
{
...
...
@@ -286,10 +289,12 @@ public class FileDAO {
fi
.
setContentMd5
(
rs
.
getString
(
"content_md5"
));
fi
.
setContentType
(
rs
.
getString
(
"content_type"
));
fi
.
setDirectory
(
rs
.
getBoolean
(
"is_directory"
));
fi
.
setJobId
(
rs
.
getString
(
"job_id"
));
fi
.
setLink
(
rs
.
getBoolean
(
"is_link"
));
fi
.
setJobId
(
rs
.
getString
(
"job_id"
));
int
locationId
=
rs
.
getInt
(
"location_id"
);
if
(!
rs
.
wasNull
())
{
fi
.
setLocationId
(
locationId
);
fi
.
setLocationType
(
rs
.
getString
(
"location_type"
));
}
fillOsPath
(
fi
,
rs
);
...
...
src/main/java/it/inaf/ia2/transfer/persistence/model/FileInfo.java
View file @
209bdccf
...
...
@@ -16,6 +16,7 @@ public class FileInfo {
private
boolean
isPublic
;
private
boolean
virtualParent
;
private
boolean
directory
;
private
boolean
link
;
private
List
<
String
>
groupRead
;
private
List
<
String
>
groupWrite
;
private
String
ownerId
;
...
...
@@ -27,6 +28,7 @@ public class FileInfo {
private
Long
contentLength
;
private
String
contentMd5
;
private
Integer
locationId
;
private
String
locationType
;
private
String
jobId
;
public
int
getNodeId
()
{
...
...
@@ -109,6 +111,14 @@ public class FileInfo {
this
.
directory
=
directory
;
}
public
boolean
isLink
()
{
return
link
;
}
public
void
setLink
(
boolean
link
)
{
this
.
link
=
link
;
}
public
boolean
hasVirtualParent
()
{
return
virtualParent
;
}
...
...
@@ -173,6 +183,14 @@ public class FileInfo {
this
.
locationId
=
locationId
;
}
public
String
getLocationType
()
{
return
locationType
;
}
public
void
setLocationType
(
String
locationType
)
{
this
.
locationType
=
locationType
;
}
public
String
getJobId
()
{
return
jobId
;
}
...
...
@@ -180,8 +198,8 @@ public class FileInfo {
public
void
setJobId
(
String
jobId
)
{
this
.
jobId
=
jobId
;
}
public
static
String
getVosParentPath
(
FileInfo
fileInfo
)
{
return
fileInfo
.
getVirtualPath
().
substring
(
0
,
fileInfo
.
getVirtualPath
().
lastIndexOf
(
"/"
));
return
fileInfo
.
getVirtualPath
().
substring
(
0
,
fileInfo
.
getVirtualPath
().
lastIndexOf
(
"/"
));
}
}
src/main/java/it/inaf/ia2/transfer/service/ArchiveService.java
View file @
209bdccf
...
...
@@ -112,7 +112,7 @@ public class ArchiveService {
continue
;
}
if
(
fileInfo
.
getLocationId
()
!=
null
&&
fileInfo
.
getLocation
Id
()
!=
uploadLocationId
)
{
if
(
fileInfo
.
getLocationId
()
!=
null
&&
"portal"
.
equals
(
fileInfo
.
getLocation
Type
())
)
{
// remote file
if
(
portalLocationUrls
==
null
)
{
portalLocationUrls
=
locationDAO
.
getPortalLocationUrls
();
...
...
src/main/java/it/inaf/ia2/transfer/service/FileCopyService.java
View file @
209bdccf
...
...
@@ -43,32 +43,32 @@ public class FileCopyService {
@Autowired
private
RestTemplate
restTemplate
;
@Autowired
private
PutFileService
putFileService
;
@Value
(
"${upload_location_id}"
)
private
int
uploadLocationId
;
public
void
copyFiles
(
String
sourceRootVosPath
,
String
destinationRootVosPath
,
String
jobId
,
TokenPrincipal
principal
)
{
LOG
.
trace
(
"copyFiles called for source {}, destination {}, from jobId \"{}\""
,
LOG
.
trace
(
"copyFiles called for source {}, destination {}, from jobId \"{}\""
,
sourceRootVosPath
,
destinationRootVosPath
,
jobId
);
// We use jobId to identify nodes created by the REST part of CopyNode
// We expect them to be locked
List
<
FileInfo
>
sources
=
fileDAO
.
getBranchFileInfos
(
sourceRootVosPath
,
jobId
);
LOG
.
debug
(
"found {} sources"
,
sources
.
size
());
if
(
sources
.
isEmpty
())
{
throw
new
NodeNotFoundException
(
sourceRootVosPath
);
}
// Set location of destinations to this file service update location
// Set location of destinations to this file service update location
// before retrieving file infos
fileDAO
.
setBranchLocationId
(
destinationRootVosPath
,
jobId
,
uploadLocationId
);
...
...
@@ -76,7 +76,7 @@ public class FileCopyService {
=
fileDAO
.
getBranchFileInfos
(
destinationRootVosPath
,
jobId
);
LOG
.
debug
(
"found {} destinations"
,
destinations
.
size
());
if
(
destinations
.
isEmpty
())
{
throw
new
NodeNotFoundException
(
destinationRootVosPath
);
}
...
...
@@ -94,7 +94,7 @@ public class FileCopyService {
principal
);
}
private
void
fillDestinations
(
List
<
FileInfo
>
sourcesFileInfos
,
List
<
FileInfo
>
destinationFileInfos
,
String
sourceRootVosPath
,
...
...
@@ -102,12 +102,12 @@ public class FileCopyService {
TokenPrincipal
principal
)
{
// it will be initialized only when necessary
Map
<
Integer
,
String
>
portalLocationUrls
=
null
;
Map
<
Integer
,
String
>
portalLocationUrls
=
null
;
for
(
FileInfo
destinationFileInfo
:
destinationFileInfos
)
{
LOG
.
trace
(
"Processing {} destination"
,
destinationFileInfo
.
getVirtualPath
());
// Cycle on files only
if
(!
destinationFileInfo
.
isDirectory
())
{
if
(!
destinationFileInfo
.
isDirectory
()
&&
!
destinationFileInfo
.
isLink
()
)
{
// Calculate source file vos path
String
correspondingSourceVosPath
=
this
.
getCorrespondingSourceVosPath
(
sourceRootVosPath
,
...
...
@@ -134,25 +134,25 @@ public class FileCopyService {
}
}
if
(
sourceFileInfo
.
getLocationId
()
!=
null
&&
sourceFileInfo
.
getLocation
Id
()
!=
uploadLocationId
)
{
if
(
sourceFileInfo
.
getLocationId
()
!=
null
&&
"portal"
.
equals
(
sourceFileInfo
.
getLocation
Type
())
)
{
// remote file
if
(
portalLocationUrls
==
null
)
{
portalLocationUrls
=
locationDAO
.
getPortalLocationUrls
();
}
String
url
=
portalLocationUrls
.
get
(
sourceFileInfo
.
getLocationId
());
// download file to destination disk path
this
.
downloadFileToDisk
(
sourceFileInfo
,
destinationFileInfo
,
this
.
downloadFileToDisk
(
sourceFileInfo
,
destinationFileInfo
,
principal
,
url
,
remainingQuota
);
}
else
{
// local file
// local file
this
.
copyLocalFile
(
sourceFileInfo
,
destinationFileInfo
,
principal
,
remainingQuota
);
}
}
}
}
}
private
String
getCorrespondingSourceVosPath
(
String
sourceRootVosPath
,
...
...
@@ -200,7 +200,7 @@ public class FileCopyService {
private
void
copyLocalFile
(
FileInfo
sourceFileInfo
,
FileInfo
destinationFileInfo
,
TokenPrincipal
tokenPrincipal
,
Long
remainingQuota
)
{
// Check permission
if
(!
authorizationService
.
isDownloadable
(
sourceFileInfo
,
tokenPrincipal
))
{
throw
PermissionDeniedException
.
forPath
(
sourceFileInfo
.
getVirtualPath
());
...
...
@@ -208,7 +208,7 @@ public class FileCopyService {
File
file
=
new
File
(
sourceFileInfo
.
getOsPath
());
LOG
.
trace
(
"Copying file: {} to {}"
,
file
.
getAbsolutePath
(),
destinationFileInfo
.
getOsPath
());
putFileService
.
copyLocalFile
(
sourceFileInfo
,
destinationFileInfo
,
remainingQuota
);
}
...
...
src/test/java/it/inaf/ia2/transfer/persistence/FileDAOTest.java
View file @
209bdccf
...
...
@@ -88,10 +88,14 @@ public class FileDAOTest {
assertEquals
(
"/home/vospace/upload/user1/file1"
,
fileInfos
.
get
(
0
).
getOsPath
());
assertEquals
(
"/home/vospace/upload/user1/file2"
,
fileInfos
.
get
(
1
).
getOsPath
());
assertEquals
(
"user"
,
fileInfos
.
get
(
1
).
getLocationType
());
assertTrue
(
fileInfos
.
get
(
2
).
isDirectory
());
assertNull
(
fileInfos
.
get
(
2
).
getLocationType
());
assertEquals
(
"/home/username1/retrieve/subdir1/file3"
,
fileInfos
.
get
(
3
).
getOsPath
());
assertEquals
(
"/home/username1/retrieve/subdir1/file4"
,
fileInfos
.
get
(
4
).
getOsPath
());
assertEquals
(
"async"
,
fileInfos
.
get
(
4
).
getLocationType
());
assertEquals
(
"portal-file"
,
fileInfos
.
get
(
5
).
getVirtualName
());
assertEquals
(
"portal"
,
fileInfos
.
get
(
5
).
getLocationType
());
}
@Test
...
...