First RichFilemanager integration with demo binds setup. (a95f8fa6) · Commits · ExaCT / Rosetta

services/slurmclustermaster/prestartup_slurmclustermaster.sh

+6 −0

Original line number	Diff line number	Diff line
		@@ -6,3 +6,9 @@ mkdir -p /shared/rosetta && chown rosetta:rosetta /shared/rosetta

		# Shared home for slurmtestuser to simulate a shared home folders filesystem
		cp -a /home_slurmtestuser_vanilla /shared/home_slurmtestuser

		# Create shared "data" and "scratch" directories
		mkdir -p /shared/scratch
		chmod 777 /shared/scratch
		mkdir -p /shared/data/users/slurmtestuser
		chown slurmtestuser:slurmtestuser /shared/data/users/slurmtestuser

services/webapp/code/rosetta/core_app/api.py

+295 −2

Original line number	Diff line number	Diff line
		import re
		import logging
		from django.http import HttpResponse
		from django.utils import timezone
		@@ -7,8 +8,9 @@ from django.conf import settings
		from rest_framework.response import Response
		from rest_framework import status, serializers, viewsets
		from rest_framework.views import APIView
		from .utils import format_exception, send_email
		from .models import Profile, Task, TaskStatuses
		from .utils import format_exception, send_email, os_shell
		from .models import Profile, Task, TaskStatuses, Computing, KeyPair
		import json

		# Setup logging
		logger = logging.getLogger(__name__)
		@@ -85,6 +87,17 @@ def rosetta_authenticate(request):
		return error401('This is a private API. Login or provide username/password or auth token')


		#==============================
		# CSRF exempt auth class
		#==============================

		from rest_framework.authentication import SessionAuthentication, BasicAuthentication

		class CsrfExemptSessionAuthentication(SessionAuthentication):

		def enforce_csrf(self, request):
		return # To not perform the csrf check previously happening


		#==============================
		# Base public API class
		@@ -325,4 +338,284 @@ print(port)
		return HttpResponse('Unknown action "{}"'.format(action))


		#==========================================
		# File manager APIs
		#==========================================






		class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI):
		"""
		get:
		Return directory listings or file contents.

		post:
		Perform actions or upload files.
		"""

		# The RichFilemanager has no CSRF support...
		authentication_classes = (CsrfExemptSessionAuthentication, BasicAuthentication)


		def prepare_command(self, command, user, computing):

		# Get user key
		user_keys = KeyPair.objects.get(user=user, default=True)


		# Get computing host
		computing_host = computing.get_conf_param('host')

		# Trick for handling Slurm.. TODO: fix me!
		if not computing_host:
		computing_host = computing.get_conf_param('master')

		computing_user = computing.get_conf_param('user')

		if not computing_host:
		raise Exception('No computing host?!')

		if not computing_user:
		raise Exception('No computing user?!')

		# Command
		command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} {}'.format(user_keys.private_key_file, computing_user, computing_host, command)

		return command

		def get_computing(self, path, request):
		# Get the computing based on the folder name # TODO: this is very weak..
		computing_resource_name = path.split('/')[1]

		# First try to get platform-level computing resource
		computing = Computing.objects.filter(name=computing_resource_name, user=None)

		# If not, fallback on the user computing name
		if not computing:
		computing = Computing.objects.filter(name=computing_resource_name, user=request.user)

		if not computing:
		raise Exception('Cannot find any computing resource named "{}"'.format(computing_resource_name+'1'))

		# Check that we had no more than one computing resource
		if len(computing) > 1:
		raise Exception('Found more than one computign resource named "{}", cannot continue!'.format(computing_resource_name))

		computing = computing[0]

		# Attach user conf in any
		computing.attach_user_conf_data(request.user)

		return computing


		def ls(self, path, user, computing, binds=[]):

		# Data container
		data = []

		# Prepare command
		command = self.prepare_command('ls -al /{}'.format(path), user, computing)

		# Execute_command
		out = os_shell(command, capture=True)
		if out.exit_code != 0:
		raise Exception(out.stderr)

		# Log
		#logger.debug('Shell exec output: "{}"'.format(out))

		out_lines = out.stdout.split('\n')

		for line in out_lines:

		# Skip total files summary line at the end
		if line.startswith('total'):
		continue

		# Set name
		name = line.split(' ')[-1]

		# Check against binds if set
		if binds:
		full_path = path + '/' + name
		show = False
		for bind in binds:
		if bind.startswith(full_path) or full_path.startswith(bind):
		show = True
		break

		if not binds or (binds and show):

		# File or directory?
		if line.startswith('d'):
		if line.split(' ')[-1] not in ['.', '..']:
		data.append({
		'id': '/{}/{}/{}/'.format(computing.name, path, name),
		'type': 'folder',
		'attributes':{
		'created': 1616415170,
		'modified': 1616415170,
		'name': name,
		'readable': 1,
		'timestamp': 1616415170,
		'writable': 1,
		'path': '/{}/{}/{}'.format(computing.name, path, name)
		}
		})
		else:
		data.append({
		'id': '/{}/{}/{}'.format(computing.name, path, name),
		'type': 'file',
		'attributes':{
		'created': 1616415170,
		'modified': 1616415170,
		'name': name,
		'readable': 1,
		'timestamp': 1616415170,
		'writable': 1,
		'path': '/{}/{}/{}'.format(computing.name, path, name)
		}
		})


		return data


		def cat(self, path, user, computing):

		# Prepare command
		command = self.prepare_command('cat /{}'.format(path), user, computing)

		# Execute_command
		out = os_shell(command, capture=True)
		if out.exit_code != 0:
		raise Exception(out.stderr)

		return out.stdout




		def _get(self, request):

		mode = request.GET.get('mode', None)
		time = request.GET.get('time', None)
		path = request.GET.get('path', None)
		_ = request.GET.get('_', None)

		# Clean for some issues that happen sometimes
		if path:
		cleaner = re.compile('(?:\/)+')
		path = re.sub(cleaner,'/',path)

		# Init
		if mode == 'initiate':
		data = json.loads('{"data":{"attributes":{"config":{"options":{"culture":"en"},"security":{"allowFolderDownload":true,"extensions":{"ignoreCase":true,"policy":"DISALLOW_LIST","restrictions":[]},"readOnly":false}}},"id":"/","type":"initiate"}}')

		elif mode == 'readfolder':

		# Base folder (computing resource-level)
		if path == '/':

		# Data container
		data = {'data':[]}

		# Get computing resources
		computings = list(Computing.objects.filter(user=None)) + list(Computing.objects.filter(user=request.user))

		for computing in computings:

		# Attach user conf in any
		computing.attach_user_conf_data(request.user)

		data['data'].append({
		'id': '/{}/'.format(computing.name),
		'type': 'folder',
		'attributes':{
		'created': 1616415170,
		'modified': 1616415170,
		'name': computing.name,
		'readable': 1,
		'timestamp': 1616415170,
		'writable': 1,
		'path': '/{}/'.format(computing.name)
		}
		})

		else:

		computing = self.get_computing(path, request)

		# If we just "entered" a computing resource, filter for its bindings
		# TODO: we can remove this and just always filter agains bind probably...
		if len(path.split('/')) == 3:
		if computing.user != request.user:
		binds = computing.get_conf_param('binds', from_sys_only=True )
		else:
		binds = computing.get_conf_param('binds')

		if binds:
		binds = binds.split(',')
		binds = [bind.split(':')[0] for bind in binds]

		# Ok, get directoris and files for this folder (always filtering by binds)
		ls_path = '/'.join(path.split('/')[2:])
		data = {'data': self.ls(ls_path, request.user, computing, binds)}

		else:
		# Ok, get directoris and files for this folder:
		ls_path = '/'.join(path.split('/')[2:])
		data = {'data': self.ls(ls_path, request.user, computing)}


		elif mode == 'download':
		logger.debug('Downloading "{}"'.format(path))
		data=''


		elif mode == 'readfile':
		logger.debug('Reading "{}"'.format(path))
		computing = self.get_computing(path, request)
		cat_path = '/'.join(path.split('/')[2:])
		data = self.cat(cat_path, request.user, computing)

		else:
		return error400('Operation "{}" not supported'.format(mode))


		return Response(data, status=status.HTTP_200_OK)


		#============================
		# POST
		#============================
		def _post(self, request):

		mode = request.POST.get('mode', None)
		time = request.POST.get('time', None)
		path = request.POST.get('path', None)
		_ = request.GET.get('_', None)


		if mode == 'savefile':
		#logger.debug('Reading "{}"'.format(path))
		#computing = self.get_computing(path, request)
		#cat_path = '/'.join(path.split('/')[2:])
		#data = self.echo(cat_path, request.user, computing)
		return error400('Operation "{}" not supported'.format(mode))

		else:
		return error400('Operation "{}" not supported'.format(mode))

		return ok200('ok')

services/webapp/code/rosetta/core_app/management/commands/core_app_populate.py

+4 −2

Original line number	Diff line number	Diff line
		@@ -211,7 +211,8 @@ class Command(BaseCommand):
		supports_singularity = True)

		ComputingSysConf.objects.create(computing = demo_remote_auth_computing,
		data = {'host': 'slurmclusterworker-one'})
		data = {'host': 'slurmclusterworker-one',
		'binds': '/shared/data/users:/shared/data/users,/shared/scratch:/shared/scratch'})

		ComputingUserConf.objects.create(user = testuser,
		computing = demo_remote_auth_computing,
		@@ -232,7 +233,8 @@ class Command(BaseCommand):

		# Create demo slurm sys computing conf
		ComputingSysConf.objects.create(computing = demo_slurm_computing,
		data = {'master': 'slurmclustermaster-main', 'default_partition': 'partition1'})
		data = {'master': 'slurmclustermaster-main', 'default_partition': 'partition1',
		'binds': '/shared/data/users:/shared/data/users,/shared/scratch:/shared/scratch'})

		# Create demo slurm user computing conf
		ComputingUserConf.objects.create(user = testuser,

services/webapp/code/rosetta/core_app/static/RichFilemanager/ReadMe.md

0 → 100755

+128 −0

Original line number	Diff line number	Diff line
		Rich Filemanager
		========================

		Rich Filemanager is an open-source file manager released under MIT license.
		Based on the @simogeo [Filemanager](https://github.com/simogeo/Filemanager), with a lot of improvements and new features:

		* Drag-and-drop support
		* Clipboard feature: copy, cut, paste, clear
		* Selectable files & folders support (mouse dragging & Ctrl key)
		* Multiple actions support for selected files & folders: move, delete, download
		* Before and After callback functions for some actions
		* Double or single click setup to open files & folders
		* Lazy loading of images thumbnails
		* Integration with AWS S3 storage
		* Integration with Imperavi Redactor WYSIWYG editor
		* Multiple & chunked uploads support - based on jQuery-File-Upload
		* New design of multiple upload window; New upload controls for each previewed file (start, abort, resume, delete, etc.)
		* Filetree: allow to open and display multiple subfolders at a time
		* Online MS Office documents viewer - based on Google Docs Viewer
		* Extended list of previewed file types via ViewerJS
		* New viewers to preview: "html" files (iframe), "md" files (markdown-it), etc.
		* CodeMirror editor now compatible with most of viewers
		* Standardized API that follows JSON API best practices to create connectors for any server-side language
		* Independent client and server sides. Can be located on different servers.
		* Independent configuration files for client and server sides.
		* Client-side configuration options may be overwritten with server-side ones using PHP connector.
		* Implemented plugins system for PHP connector (server-based)
		* Added new "Type" column in the list view
		* Added ability to limit max size of the storage (root folder)
		* Implemented natural sorting on the client-side

		To see the full list check out [changelog file](https://github.com/servocoder/RichFilemanager/blob/master/changelog).


		Demo
		----

		Filemanager live example: http://fm.devale.pro


		Compatibility
		-------------

		Filemanager is designed to interact with a number of programming languages via [connectors](https://github.com/servocoder/RichFilemanager/tree/master/connectors).
		The actual connectors are: PHP, Java, ASHX, ASP, NodeJs & Python 3 Flask.
		You are still able you to download unsupported v0.8 from [archive](https://github.com/simogeo/Filemanager/archive/v0.8.zip) (CFM, lasso, PL, JSP and Python WSGI)

		Browser compatibility:

		* IE9+
		* Chrome
		* FireFox
		* Opera


		Installation and Setup
		----------------------

		* [Deploy and setup RichFilemanager on your website](https://github.com/servocoder/RichFilemanager/wiki/Deploy-and-setup)
		* [Discover complete configuration guidelines](https://github.com/servocoder/RichFilemanager/wiki/Configuration-options)


		Documentation
		-------------

		Filemanager is highly documented on the [wiki pages](https://github.com/servocoder/RichFilemanager/wiki). API, see below.


		Main features
		-------------

		* Available in more than 20 languages.
		* [Highly customizable](https://github.com/servocoder/RichFilemanager/wiki/Configuration-options)
		* Can work as standalone application
		* Easy integration with WYSIWYG editors like CKEditor, TinyMCE, Imperavi Redactor and so on.
		* Easy integration with [AWS S3 storage](https://github.com/servocoder/RichFilemanager-PHP) to manipulate your files on remote S3 server.
		* Easy integration with [colorbox jquery plugin](https://github.com/servocoder/RichFilemanager/wiki/How-to-use-the-filemanager-with-colorbox) or [HTML simple textfield](https://github.com/servocoder/RichFilemanager/wiki/How-to-use-the-filemanager-from-a-simple-textfield)
		* 2 view modes: grid and list
		* Drag-and-drop support
		* Clipboard feature: copy, cut, paste, clear
		* Single file actions: upload, modify, move, delete, download
		* Single folder actions: create, modify, move, delete, download (zip archive)
		* Selectable support for files & folders (mouse dragging & Ctrl key)
		* Multiple actions support for selected files & folders: move, delete, download
		* Support user permissions - based on session
		* Handle system permissions
		* Ability to pass config user file in URL
		* Multiple & chunked uploads support - based on [jQuery-File-Upload](https://github.com/blueimp/jQuery-File-Upload)
		* Online text / code edition - based on [codeMirror](http://codemirror.net/)
		* Online PDF & OpenOffice documents viewer - based on [viewerJS](http://viewerjs.org/)
		* Online MS Office documents viewer - based on [Google Docs Viewer](http://docs.google.com/viewer/)
		* Several server-side language connectors available. PHP, Java, ASHX, ASP, NodeJs & Python 3 Flask up-to-date
		* Standardized API that follows JSON API best practices to create connectors for any server-side language
		* Independent client and server sides. Can be located on different servers.
		* [Opening a given folder](https://github.com/servocoder/RichFilemanager/wiki/How-to-open-a-given-folder-different-from-root-folder-when-opening-the-filemanager)
		* [Opening exclusively a given folder](https://github.com/servocoder/RichFilemanager/wiki/How-to-open-%28exclusively%29-a-given-subfolder)
		* [Passing parameters to the FM](https://github.com/servocoder/RichFilemanager/wiki/Passing-parameters-to-the-FM)
		* File types and patterns restrictions
		* Video and audio player relying on web browser capabilities
		* Textbox Search filter
		* Thumbnails generation
		* Image auto-resize
		* File size limit
		* File exclusion based on name and patterns
		* Prevent files overwriting (or not)
		* Copy direct file URL
		* [CSS Themes](https://github.com/servocoder/RichFilemanager/wiki/Create-your-own-theme) - Please, share your themes with others !
		* and more ...


		Screenshot
		-------------

		![Filemanager Screenshot](http://image.prntscr.com/image/36ed7f7531454f75b5462764f02b2cbd.png)


		Contribution
		------------

		Any contribution is greatly appreciated.
		You can become a maintainer for any of existent connectors, or create new one for your server side language.
		Check the details in [API](https://github.com/servocoder/RichFilemanager/wiki/API) section.


		MIT LICENSE
		-----------

		Released under the [MIT license](http://opensource.org/licenses/MIT).
		No newline at end of file

services/webapp/code/rosetta/core_app/static/RichFilemanager/bower.json

0 → 100644

+27 −0

Original line number	Diff line number	Diff line
		{
		"name": "rich-filemanager",
		"description": "Highly customizable open-source file manager",
		"main": "scripts/filemanager.js",
		"authors": [
		"Pavel Solomienko"
		],
		"license": "MIT",
		"keywords": [
		"filemanager",
		"fileupload",
		"manager",
		"upload",
		"file",
		"rich",
		"js",
		"php"
		],
		"homepage": "https://github.com/servocoder/RichFilemanager",
		"ignore": [
		"*/.",
		"node_modules",
		"bower_components",
		"test",
		"tests"
		]
		}