Initial import

.gitignore

@@ -0,0 +1,4 @@
+venv
+__pycache__/
+
+*.egg-info

README.md

@@ -0,0 +1,13 @@
+This is a simple file storage and permanent archive service.
+It allows people to drop off files which are then assigned a uuid to link to.
+
+=Install dependencies=
+
+export VENV=/path/to/venv
+python3 -m venv $VENV
+$VENV/bin/pip install -e .
+
+=Run the application stand-alone=
+
+$VENV/bin/pserve settings.ini --reload
+

filestorage/__init__.py

@@ -0,0 +1,61 @@
+import os
+import shelve
+
+from wsgiref.simple_server import make_server
+from pyramid.config import Configurator
+
+
+"""
+Main entrance point that returns a wsgi application
+"""
+def main(global_config, **settings):
+    config = Configurator(settings=settings)
+
+    # Home view when visiting the app on /
+    config.add_route('home', '/')
+
+    # REST-API view behind /files
+    config.add_route('files', '/files')
+    config.add_route('files_uuid', '/files/{file_id}')
+    config.add_route('file_details', '/files/{file_id}/info')
+    config.add_route('file_uuid_key', '/files/{file_id}/{file_key}')
+
+    # Route for static files
+    config.add_static_view(name='static', path='filestorage:static')
+
+    # Add pyramid_jinja2 module to app config for web_views
+    config.include('pyramid_jinja2')
+
+    # Do a config scan to get all the views
+    config.scan()
+
+    # Make sure the infrastructure (folders, database) are set up
+    setup_infrastructure(config.registry.settings)
+
+    # Finally build and return the app
+    app = config.make_wsgi_app()
+    return app
+
+
+"""
+Checkup routine to make sure folders and database are fine
+"""
+def setup_infrastructure(settings):
+    # Check temp folder
+    tmp_folder = settings['path_tmp']
+    if not os.path.exists(tmp_folder):
+        os.makedirs(tmp_folder)
+    if not os.access(tmp_folder, os.W_OK):
+        raise PermissionError('Unable to write to temporary folder: %s' % tmp_folder)
+    # Check file storage folder
+    path_file_storage = settings['path_file_storage']
+    if not os.path.exists(path_file_storage):
+        os.makedirs(path_file_storage)
+    if not os.access(path_file_storage, os.W_OK):
+        raise PermissionError('Unable to write to file storage folder: %s' % path_file_storage)
+    # Make sure shelve database works (and exists)
+    path_database = settings['path_database']
+    db = shelve.open(path_database, flag='c')
+    db.close()
+    pass
+

filestorage/api_views.py

@@ -0,0 +1,134 @@
+import datetime
+import hashlib
+import os
+import shelve
+import shutil
+import uuid
+
+from pyramid.view import view_defaults
+from pyramid.view import view_config
+from pyramid.response import Response, FileResponse
+
+
+"""
+This REST API provides a file resource to upload, update, download
+and delete files.
+"""
+@view_defaults(route_name='files', renderer='json')
+class RESTView(object):
+    def __init__(self, request):
+        self.request = request
+        self.path_db = request.registry.settings['path_database']
+        self.path_tmp = request.registry.settings['path_tmp']
+        self.path_file_storage = request.registry.settings['path_file_storage']
+
+    def db_fetch_file(self, file_id):
+        with shelve.open(self.path_db, flag='r') as db:
+            if file_id not in db:
+                raise LookupError('File with uuid %s does not exist!' % uuid)
+            else:
+                return db[file_id]
+
+    def db_add_file(self, file_id, data):
+        with shelve.open(self.path_db, flag='w') as db:
+            if file_id in db:
+                raise Error('File with uuid %s already exists!' % uuid)
+            else:
+                db[file_id] = data
+
+    def db_update_file(self, file_id, data):
+        with shelve.open(self.path_db, flag='w') as db:
+            if file_id not in db:
+                raise LookupError('File with uuid %s does not exist, cannot update!' % uuid)
+            else:
+                db[file_id] = data
+
+    def db_delete_file(self, file_id):
+        with shelve.open(self.path_db, flag='w') as db:
+            if file_id in db:
+                del db[file_id]
+            else:
+                raise LookupError('File with uuid %s does not exist, cannot delete!' % uuid)
+
+    """
+    Helper method to calculate hash of file
+    """
+    def hash_file(self, filename, hasher):
+        BLOCKSIZE = 65536
+        with open(filename, 'rb') as infile:
+            buf = infile.read(BLOCKSIZE)
+            while len(buf) > 0:
+                hasher.update(buf)
+                buf = infile.read(BLOCKSIZE)
+        return hasher.hexdigest()
+
+    @view_config(request_method='PUT')
+    def put(self):
+        try:
+            # Create a fresh uuid for the new file
+            file_id = str(uuid.uuid4())
+            # Use PUT /files to create a new file
+            input_file = self.request.POST['file'].file
+            # Write the data into a temporary file
+            temp_file_path = os.path.join(self.path_tmp, '%s' % file_id) + '~'
+            input_file.seek(0)
+            with open(temp_file_path, 'wb') as output_file:
+                shutil.copyfileobj(input_file, output_file)
+            # Now that we know the file has been fully saved to disk move it into place.
+            target_file_path = os.path.join(self.path_file_storage, '%s' % file_id)
+            os.rename(temp_file_path, target_file_path)
+            # Calculate file hash
+            filehash_md5 = self.hash_file(target_file_path, hashlib.md5())
+            filehash_sha1 = self.hash_file(target_file_path, hashlib.sha1())
+            filehash_sha256 = self.hash_file(target_file_path, hashlib.sha256())
+            # Gather all data about the file
+            file_data = dict(uuid=file_id,name=self.request.POST['file'].filename, key=str(uuid.uuid4()), create_utc=str(datetime.datetime.utcnow()), sha256=filehash_sha256, sha1=filehash_sha1, md5=filehash_md5)
+            # Store file data in database
+            self.db_add_file(file_id, file_data)
+            # Hopefully everything worked out, so return the data from the file
+            return file_data
+        except:
+            return dict(success=False)
+
+    @view_config(request_method='GET', route_name='files_uuid')
+    def get(self):
+        # Use GET /files/<uuid> to retrieve the file
+        file_id = self.request.matchdict['file_id']
+        try:
+            file_data = self.db_fetch_file(file_id)
+            external_filename = file_data['name']
+            file_response = FileResponse(os.path.join(self.path_file_storage, '%s' % file_id))
+            file_response.content_disposition = 'attachment; filename=%s' % external_filename
+            return file_response
+        except:
+            return dict(success=False)
+
+    @view_config(request_method='GET', route_name='file_details')
+    def get_info(self):
+        # Use GET /files/<uuid>/info to get file details
+        file_id = self.request.matchdict['file_id']
+        try:
+            file_data = self.db_fetch_file(file_id)
+            del file_data['key'] # Remove the secret key only the creator is supposed to know
+            return file_data
+        except:
+            return dict(success=False)
+
+    @view_config(request_method='DELETE', route_name='file_uuid_key')
+    def delete(self):
+        # Use DELETE /files/<uuid>/<key> to delete file
+        file_id = self.request.matchdict['file_id']
+        file_key = self.request.matchdict['file_key']
+        try:
+            file_data = self.db_fetch_file(file_id)
+        except:
+            return dict(success=False)
+        if file_data['key'] == file_key:
+            # Delete file and db entry
+            file_path = os.path.join(self.path_file_storage, '%s' % file_id)
+            os.remove(file_path)
+            self.db_delete_file(file_id)
+            return dict(success=True)
+        else:
+            return dict(success=False)
+

filestorage/static/app.css

@@ -0,0 +1,4 @@
+body {
+    margin: 2em;
+    font-family: sans-serif;
+}

filestorage/static/app.js

@@ -0,0 +1 @@
+# TODO: Implement the jQuery REST API client!

filestorage/templates/home.jinja2

@@ -0,0 +1,29 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <title>Filestorage Service</title>
+    <meta charset="utf-8">
+    <link rel="stylesheet" href="{{ request.static_url('filestorage:static/app.css') }}"/>
+    <script type="text/javascript" src="{{ request.static_url('filestorage:static/jquery-3.1.1.min.js') }}"></script>
+    <script type="text/javascript" src="{{ request.static_url('filestorage:static/app.js') }}"></script>
+</head>
+<body>
+  <h1>Welcome!</h1>
+  <p>
+    This site enables users to upload files for temporary or permanent access.
+    Since the PUT method of the API is used to upload files, please enable javascript.
+  </p>
+  <p>
+    Use this command to upload a file using curl:
+    <pre>curl {{ request.route_url('files') }} -X PUT -F "file=@/path/to/your/file"</pre>
+  </p>
+  <p>
+    Use this form to upload a file! (TODO: Implement with jQuery, properly display results)
+    <form action="/files" method="PUT" accept-charset="utf-8" enctype="multipart/form-data">
+      <label for="file">File</label>
+      <input id="file" name="file" type="file" value="" />
+      <input type="submit" value="Upload!" />
+    </form>
+  </p>
+</body>
+</html>

filestorage/web_views.py

@@ -0,0 +1,44 @@
+import cgi
+
+from pyramid.httpexceptions import HTTPFound
+from pyramid.response import Response
+from pyramid.view import view_config
+
+
+@view_config(route_name='home', renderer='templates/home.jinja2')
+def home_view(request):
+    return dict()
+    # This is how params can be passed to the renderer
+    #return dict(name=request.matchdict['name'])
+
+"""
+# /howdy?name=alice which links to the next view
+@view_config(route_name='hello')
+def hello_view(request):
+    name = request.params.get('name', 'No Name')
+    body = '<p>Hi %s, this <a href="/goto">redirects</a></p>'
+    # cgi.escape to prevent Cross-Site Scripting (XSS) [CWE 79]
+    return Response(body % cgi.escape(name))
+
+
+# /goto which issues HTTP redirect to the last view
+@view_config(route_name='redirect')
+def redirect_view(request):
+    return HTTPFound(location="/problem")
+
+
+# /problem which causes a site error
+@view_config(route_name='exception')
+def exception_view(request):
+    raise Exception()
+
+# /get which shows a file by its uuid
+@view_config(route_name='get_file')
+def get_file_view(request):
+    body = 'Param: $(uuid)' % request.matchdict
+    return Response(body)
+
+view_config(route_name='home', renderer='json')
+def hello_json(request):
+    return [1, 2, 3]
+"""

settings.ini

@@ -0,0 +1,10 @@
+[app:main]
+use = egg:filestorage
+path_tmp = /tmp
+path_file_storage = /tmp/bucket
+path_database = /tmp/filestorage.shelve
+
+[server:main]
+use = egg:pyramid#wsgiref
+host = 127.0.0.1
+port = 6543

setup.py

@@ -0,0 +1,14 @@
+from setuptools import setup
+
+requires = [
+    'pyramid',
+    'pyramid_jinja2',
+]
+
+setup(name='filestorage',
+      install_requires=requires,
+      entry_points="""\
+      [paste.app_factory]
+      main = filestorage:main
+      """,
+)