From ef672ed6b16ee5b2464412dbc79bd2f5a07fd635 Mon Sep 17 00:00:00 2001 From: Ryan Smith <0ryansmith1994@gmail.com> Date: Mon, 17 Aug 2015 10:26:38 +0100 Subject: [PATCH 01/19] Improves performance of migration. --- ...08_12_150203_ConsistentForeignKeyNames.php | 93 +++++++++---------- 1 file changed, 43 insertions(+), 50 deletions(-) diff --git a/app/database/migrations/2015_08_12_150203_ConsistentForeignKeyNames.php b/app/database/migrations/2015_08_12_150203_ConsistentForeignKeyNames.php index 9d43bf4426..8027673441 100644 --- a/app/database/migrations/2015_08_12_150203_ConsistentForeignKeyNames.php +++ b/app/database/migrations/2015_08_12_150203_ConsistentForeignKeyNames.php @@ -5,63 +5,56 @@ class ConsistentForeignKeyNames extends Migration { public function up() { - $this->chunkKeyRenaming(new DocumentAPI, 'lrs', 'lrs_id'); - $this->chunkKeyRenaming(new Statement, 'lrs._id', 'lrs_id'); - $this->chunkKeyRenaming(new Lrs, 'owner._id', 'owner_id'); - $this->chunkKeyRenaming(new Report, 'lrs', 'lrs_id'); - $this->chunkKeyRenaming(new Export, 'lrs', 'lrs_id'); - } - - public function down() { - $this->chunkMongoIdRemoval(new DocumentAPI, 'lrs', 'lrs_id'); - $this->chunkMongoIdRemoval(new Statement, 'lrs._id', 'lrs_id'); - $this->chunkMongoIdRemoval(new Lrs, 'owner._id', 'owner_id'); - $this->chunkMongoIdRemoval(new Report, 'lrs', 'lrs_id'); - $this->chunkMongoIdRemoval(new Export, 'lrs', 'lrs_id'); - } + $db = \DB::getMongoDB(); - private function chunkKeyRenaming($model, $old_name, $new_name) { - $this->chunkModelMigration($model, $this->renameKey($old_name, $new_name)); - } + Lrs::get()->each(function (Lrs $lrs) use ($db) { + $convertToMongoId = function ($value) { + return new \MongoId($value); + }; + $this->changeForeignKey($db->statements, 'lrs._id', 'lrs_id', $lrs->_id, $convertToMongoId); + $this->changeForeignKey($db->documentapi, 'lrs', 'lrs_id', $lrs->_id, $convertToMongoId); + $this->changeForeignKey($db->reports, 'lrs', 'lrs_id', $lrs->_id, $convertToMongoId); + $this->changeForeignKey($db->exports, 'lrs', 'lrs_id', $lrs->_id, $convertToMongoId); - private function chunkMongoIdRemoval($model, $old_name, $new_name) { - $this->chunkModelMigration($model, $this->removeMongoId($old_name, $new_name)); - } + $lrs->owner_id = $convertToMongoId($lrs->owner['_id']); + $lrs->save(); - private function removeMongoId($old_name, $new_name) { - return function ($model) use ($old_name, $new_name) { - $value = $model->$new_name; - $model = $this->setKey($model, explode('.', $old_name), 0, (string) $value); - $model->save(); - }; - } + echo 'Models for "'.$lrs->title.'" converted.'.PHP_EOL; + }); - private function setKey($model, $keys, $key_index, $value) { - if ($key_index < count($keys) - 1) { - $model->{$keys[$key_index]} = (object) []; - $this->setKey($model->{$keys[$key_index]}, $keys, $key_index + 1, $value); - } else { - $model->{$keys[$key_index]} = $value; - } - return $model; - } + echo 'All finished, hopefully!'.PHP_EOL; + } - private function renameKey($old_name, $new_name) { - return function ($model) use ($old_name, $new_name) { - $value = array_reduce(explode('.', $old_name), function ($value, $key) { - return is_object($value) ? $value->{$key} : $value[$key]; - }, $model); - $model->$new_name = new \MongoId($value); - $model->save(); - }; + private function changeForeignKey($collection, $old_key, $new_key, $old_value, $modifier) { + $collection->update([ + $old_key => $old_value + ], [ + '$set' => [ + $new_key => $modifier($old_value) + ] + ], [ + 'multiple' => true + ]); } - private function chunkModelMigration($model, Callable $migration) { - $model->chunk(1000, function ($models) use ($migration) { - foreach ($models as $model){ - $migration($model); - } - echo count($models) . ' converted.'.PHP_EOL; + public function down() { + $db = \DB::getMongoDB(); + + Lrs::get()->each(function (Lrs $lrs) use ($db) { + $convertToString = function ($value) { + return (string) $value; + }; + $this->changeForeignKey($db->statements, 'lrs_id', 'lrs._id', $lrs->_id, $convertToString); + $this->changeForeignKey($db->documentapi, 'lrs_id', 'lrs', $lrs->_id, $convertToString); + $this->changeForeignKey($db->reports, 'lrs_id', 'lrs', $lrs->_id, $convertToString); + $this->changeForeignKey($db->exports, 'lrs_id', 'lrs', $lrs->_id, $convertToString); + + $lrs->owner = [ + '_id' => $convertToString($lrs->owner_id) + ]; + $lrs->save(); + + echo 'Models for "'.$lrs->title.'" converted.'.PHP_EOL; }); echo 'All finished, hopefully!'.PHP_EOL; From 0199b228d16b4d00d4991b7e9b562b228a851b76 Mon Sep 17 00:00:00 2001 From: Ryan Smith <0ryansmith1994@gmail.com> Date: Tue, 18 Aug 2015 08:27:57 +0100 Subject: [PATCH 02/19] Adds FlySystem. --- composer.json | 3 +- composer.lock | 89 ++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 87 insertions(+), 5 deletions(-) diff --git a/composer.json b/composer.json index 25e8986c2b..b73e93ec64 100644 --- a/composer.json +++ b/composer.json @@ -43,7 +43,8 @@ "learninglocker/statementfactory": "0.0.x", "bshaffer/oauth2-server-php": "1.7.x", "bshaffer/oauth2-server-httpfoundation-bridge": "1.1.x", - "jenssegers/mongodb-session": "~1.0" + "jenssegers/mongodb-session": "~1.0", + "league/flysystem": "~1.0" }, "require-dev": { "phpunit/phpunit": "4.1.*" diff --git a/composer.lock b/composer.lock index 2aefb6ed90..ad78544c73 100644 --- a/composer.lock +++ b/composer.lock @@ -1,10 +1,10 @@ { "_readme": [ "This file locks the dependencies of your project to a known state", - "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file", + "Read more about it at http://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file", "This file is @generated automatically" ], - "hash": "5c3730d30a2f50f10a2943aa9f48ddcb", + "hash": "ae8e6f219e167ae88fb097cd215acbe8", "packages": [ { "name": "andywer/js-localization", @@ -251,12 +251,12 @@ "version": "1.0.2", "source": { "type": "git", - "url": "https://github.com/ClassPreloader/ClassPreloader.git", + "url": "https://github.com/mtdowling/ClassPreloader.git", "reference": "2c9f3bcbab329570c57339895bd11b5dd3b00877" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/ClassPreloader/ClassPreloader/zipball/2c9f3bcbab329570c57339895bd11b5dd3b00877", + "url": "https://api.github.com/repos/mtdowling/ClassPreloader/zipball/2c9f3bcbab329570c57339895bd11b5dd3b00877", "reference": "2c9f3bcbab329570c57339895bd11b5dd3b00877", "shasum": "" }, @@ -741,6 +741,87 @@ ], "time": "2015-02-11 20:37:15" }, + { + "name": "league/flysystem", + "version": "1.0.11", + "source": { + "type": "git", + "url": "https://github.com/thephpleague/flysystem.git", + "reference": "c16222fdc02467eaa12cb6d6d0e65527741f6040" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/thephpleague/flysystem/zipball/c16222fdc02467eaa12cb6d6d0e65527741f6040", + "reference": "c16222fdc02467eaa12cb6d6d0e65527741f6040", + "shasum": "" + }, + "require": { + "php": ">=5.4.0" + }, + "require-dev": { + "ext-fileinfo": "*", + "mockery/mockery": "~0.9", + "phpspec/phpspec": "^2.2", + "phpspec/prophecy-phpunit": "~1.0", + "phpunit/phpunit": "~4.1" + }, + "suggest": { + "ext-fileinfo": "Required for MimeType", + "league/flysystem-aws-s3-v2": "Allows you to use S3 storage with AWS SDK v2", + "league/flysystem-aws-s3-v3": "Allows you to use S3 storage with AWS SDK v3", + "league/flysystem-azure": "Allows you to use Windows Azure Blob storage", + "league/flysystem-cached-adapter": "Flysystem adapter decorator for metadata caching", + "league/flysystem-copy": "Allows you to use Copy.com storage", + "league/flysystem-dropbox": "Allows you to use Dropbox storage", + "league/flysystem-eventable-filesystem": "Allows you to use EventableFilesystem", + "league/flysystem-rackspace": "Allows you to use Rackspace Cloud Files", + "league/flysystem-sftp": "Allows you to use SFTP server storage via phpseclib", + "league/flysystem-webdav": "Allows you to use WebDAV storage", + "league/flysystem-ziparchive": "Allows you to use ZipArchive adapter" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.1-dev" + } + }, + "autoload": { + "psr-4": { + "League\\Flysystem\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Frank de Jonge", + "email": "info@frenky.net" + } + ], + "description": "Filesystem abstraction: Many filesystems, one API.", + "keywords": [ + "Cloud Files", + "WebDAV", + "abstraction", + "aws", + "cloud", + "copy.com", + "dropbox", + "file systems", + "files", + "filesystem", + "filesystems", + "ftp", + "rackspace", + "remote", + "s3", + "sftp", + "storage" + ], + "time": "2015-07-28 20:41:58" + }, { "name": "learninglocker/statementfactory", "version": "v0.0.2", From 8bf66abd3cce954f0facb4ab15a09118461edbc4 Mon Sep 17 00:00:00 2001 From: Ryan Smith <0ryansmith1994@gmail.com> Date: Tue, 18 Aug 2015 16:34:38 +0100 Subject: [PATCH 03/19] Adds support for FlySystem. --- app/controllers/xapi/DocumentController.php | 17 ++++++----- .../Document/EloquentDocumentRepository.php | 1 - .../repository/RepositoryServiceProvider.php | 6 +++- .../repository/Statement/FileAttacher.php | 19 ++++++------- app/models/DocumentAPI.php | 28 ++++++++----------- q | 10 ------- 6 files changed, 33 insertions(+), 48 deletions(-) delete mode 100644 q diff --git a/app/controllers/xapi/DocumentController.php b/app/controllers/xapi/DocumentController.php index f1a106b597..f937cddcac 100644 --- a/app/controllers/xapi/DocumentController.php +++ b/app/controllers/xapi/DocumentController.php @@ -1,8 +1,8 @@ content, 200, $headers); default: - return \Response::download( - $document->getFilePath(), - $document->content, - $headers - ); + $stream = (new FileRepository)->stream($document->getFilePath(), []); + return \Response::stream(function () use ($stream) { + fpassthru($stream); + }, 200, $headers); } } } diff --git a/app/locker/repository/Document/EloquentDocumentRepository.php b/app/locker/repository/Document/EloquentDocumentRepository.php index 9e887de0e7..be1f73f151 100644 --- a/app/locker/repository/Document/EloquentDocumentRepository.php +++ b/app/locker/repository/Document/EloquentDocumentRepository.php @@ -387,7 +387,6 @@ public function storeActivityDoc( $options, $data, $updated, $method ){ $document->updated_at = new Carbon($updated); $document->setContent( $data['content_info'], $method ); //set the content for the document - if( $document->save() ){ return $document; } diff --git a/app/locker/repository/RepositoryServiceProvider.php b/app/locker/repository/RepositoryServiceProvider.php index 7ca0616e94..b9ac6b1883 100644 --- a/app/locker/repository/RepositoryServiceProvider.php +++ b/app/locker/repository/RepositoryServiceProvider.php @@ -46,7 +46,11 @@ public function register(){ 'Locker\Repository\Export\Repository', 'Locker\Repository\Export\EloquentRepository' ); + $this->app->bind( + 'Locker\Repository\File\Repository', + 'Locker\Repository\File\FlyRepository' + ); } -} \ No newline at end of file +} diff --git a/app/locker/repository/Statement/FileAttacher.php b/app/locker/repository/Statement/FileAttacher.php index c48cffd3be..afa7ec74f7 100644 --- a/app/locker/repository/Statement/FileAttacher.php +++ b/app/locker/repository/Statement/FileAttacher.php @@ -1,8 +1,8 @@ getDir($opts); - if (!is_dir($dir) && count($attachments > 0) && !empty($attachments)) { - mkdir($dir, 0775, true); - } foreach ($attachments as $attachment) { if (!in_array($attachment->hash, $hashes)) throw new Exceptions\Exception( @@ -28,8 +25,8 @@ public function store(array $attachments, array $hashes, StoreOptions $opts) { 'This file type cannot be supported' ); - $file = $attachment->hash.'.'.$ext; - file_put_contents($dir.$file, $attachment->content); + $filename = $attachment->hash.'.'.$ext; + (new FileRepository)->update($dir.$filename, ['content' => $attachment->content], []); } } @@ -50,7 +47,7 @@ public function index(array $statements, IndexOptions $opts) { return (object) [ 'content_type' => $attachment->contentType, 'hash' => $attachment->sha2, - 'content' => file_get_contents($dir.$filename) + 'content' => (new FileRepository)->show($dir.$filename, []) ]; }, isset($statement->attachments) ? $statement->attachments : [])); } @@ -73,6 +70,6 @@ private function getExt($content_type) { * @return String */ private function getDir(Options $opts) { - return Helpers::getEnvVar('LOCAL_FILESTORE').'/'.$opts->getOpt('lrs_id').'/attachments/'; + return $opts->getOpt('lrs_id').'/attachments/'; } } diff --git a/app/models/DocumentAPI.php b/app/models/DocumentAPI.php index 87b45c99fb..8db30c0fe2 100644 --- a/app/models/DocumentAPI.php +++ b/app/models/DocumentAPI.php @@ -7,8 +7,9 @@ use Jenssegers\Mongodb\Model as Eloquent; use Symfony\Component\HttpFoundation\File\UploadedFile; use Locker\Repository\Document\FileTypes; -use \Locker\Helpers\Helpers as Helpers; -use \Locker\Helpers\Exceptions as Exceptions; +use Locker\Helpers\Helpers as Helpers; +use Locker\Helpers\Exceptions as Exceptions; +use Locker\Repository\File\FlyRepository as FileRepository; class DocumentAPI extends Eloquent { protected $collection = 'documentapi'; @@ -38,12 +39,6 @@ private function putContent($content, $contentType) { } } - private function ammendmentError() { - return new Exceptions\Exception( - "Cannot amend existing {$this->contentType} document with a string" - ); - } - private function postContent($content, $contentType) { if( $this->exists ){ @@ -94,13 +89,19 @@ private function saveDocument($content, $contentType) { $origname = $content->getClientOriginalName(); $parts = pathinfo($origname); $filename = Str::slug(Str::lower($parts['filename'])).'-'.time().'.'.$parts['extension']; - $content->move($dir, $filename); + + // Stores the file in a temporary location before writing it to the FileRepository. + $tmp_location = __DIR__.'/../../uploads/tmp'; + $content->move($tmp_location, $filename); + $data = file_get_contents($tmp_location.'/'.$filename); + (new FileRepository)->update($dir.$filename, ['content' => $data], []); + unlink($tmp_location.'/'.$filename); } else { $ext = array_search($contentType, FileTypes::getMap()); $filename = time().'_'.mt_rand(0,1000).($ext !== false ? '.'.$ext : ''); - $size = file_put_contents($dir.$filename, $content); + $size = (new FileRepository)->update($dir.$filename, ['content' => $content], []); if ($size === false) throw new Exceptions\Exception('There was an issue saving the content'); } @@ -136,12 +137,7 @@ public function setContent( $content_info, $method){ } public function getContentDir(){ - $dir = Helpers::getEnvVar('LOCAL_FILESTORE').'/'.$this->lrs_id.'/documents/'; - if( !file_exists($dir) ){ - mkdir( $dir, 0774, true ); - } - - return $dir; + return $this->lrs_id.'/documents/'; } public function getFilePath(){ diff --git a/q b/q deleted file mode 100644 index c4de88ad74..0000000000 --- a/q +++ /dev/null @@ -1,10 +0,0 @@ - Ryan Smith  Fixes owners. 2 hours ago - Ryan Smith  Fixes users. 2 hours ago - Ryan Smith  Refactors to lrs_id. 2 hours ago - Ryan Smith  Begins migration of foreign keys. 4 hours ago - daniel-abbey  Update VERSION 6 days ago - daniel-abbey  Merge pull request #707 from LearningLocker/issue/config 6 days ago - daniel-abbey  Merge pull request #706 from LearningLocker/feature/dailylogs2 6 days ago - daniel-abbey  Merge pull request #705 from LearningLocker/feature/password_reset 6 days ago - Ryan Smith  Only ignores php files in local config. 9 days ago - Ryan Smith  Attempts to improve config. 9 days ago \ No newline at end of file From a9d010f6495a5e2d31cc99cac1bc4b432cb7a494 Mon Sep 17 00:00:00 2001 From: Ryan Smith <0ryansmith1994@gmail.com> Date: Tue, 18 Aug 2015 16:34:48 +0100 Subject: [PATCH 04/19] Adds support for FlySystem. --- app/locker/repository/File/FlyRepository.php | 42 ++++++++++++++++++++ app/locker/repository/File/Repository.php | 4 ++ 2 files changed, 46 insertions(+) create mode 100644 app/locker/repository/File/FlyRepository.php create mode 100644 app/locker/repository/File/Repository.php diff --git a/app/locker/repository/File/FlyRepository.php b/app/locker/repository/File/FlyRepository.php new file mode 100644 index 0000000000..7572777956 --- /dev/null +++ b/app/locker/repository/File/FlyRepository.php @@ -0,0 +1,42 @@ +filesystem = new Filesystem($adapter); + } + + public function index(array $opts) { + throw new \BadMethodCallException(); + } + + public function show($id, array $opts) { + \Log::info($id); + \Log::info(Helpers::getEnvVar('LOCAL_FILESTORE')); + return $this->filesystem->read($id); + } + + public function destroy($id, array $opts) { + $this->filesystem->delete($id); + return true; + } + + public function store(array $data, array $opts) { + throw new \BadMethodCallException(); + } + + public function update($id, array $data, array $opts) { + $this->filesystem->put($id, $data['content']); + return $data; + } + + public function stream($id, array $opts) { + return $this->filesystem->readStream($id); + } + +} diff --git a/app/locker/repository/File/Repository.php b/app/locker/repository/File/Repository.php new file mode 100644 index 0000000000..2bacfa02a8 --- /dev/null +++ b/app/locker/repository/File/Repository.php @@ -0,0 +1,4 @@ + Date: Tue, 18 Aug 2015 16:35:34 +0100 Subject: [PATCH 05/19] Removes logging. --- app/locker/repository/File/FlyRepository.php | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/locker/repository/File/FlyRepository.php b/app/locker/repository/File/FlyRepository.php index 7572777956..cbbfc14494 100644 --- a/app/locker/repository/File/FlyRepository.php +++ b/app/locker/repository/File/FlyRepository.php @@ -16,8 +16,6 @@ public function index(array $opts) { } public function show($id, array $opts) { - \Log::info($id); - \Log::info(Helpers::getEnvVar('LOCAL_FILESTORE')); return $this->filesystem->read($id); } From 07125af4ef7d22e279548d7178ba2b130da4d707 Mon Sep 17 00:00:00 2001 From: Ryan Smith <0ryansmith1994@gmail.com> Date: Wed, 19 Aug 2015 15:36:34 +0100 Subject: [PATCH 06/19] Adds support for Rackspace FlySystem adapter. --- .env.php | 6 +- app/config/session.php | 2 +- app/controllers/xapi/DocumentController.php | 4 +- .../2015_03_31_130813_attachment_rename.php | 2 +- app/locker/repository/File/Factory.php | 14 + app/locker/repository/File/FlyRepository.php | 10 +- .../repository/File/LocalFlyRepository.php | 11 + .../File/RackspaceFlyRepository.php | 19 ++ app/locker/repository/File/readme.md | 29 +++ .../repository/RepositoryServiceProvider.php | 4 - .../repository/Statement/FileAttacher.php | 6 +- app/models/DocumentAPI.php | 6 +- app/tests/routes/StatementAttachmentTest.php | 2 +- composer.json | 3 +- composer.lock | 244 +++++++++++++++++- 15 files changed, 335 insertions(+), 27 deletions(-) create mode 100644 app/locker/repository/File/Factory.php create mode 100644 app/locker/repository/File/LocalFlyRepository.php create mode 100644 app/locker/repository/File/RackspaceFlyRepository.php create mode 100644 app/locker/repository/File/readme.md diff --git a/.env.php b/.env.php index 3fe3d27584..5de895fd5b 100644 --- a/.env.php +++ b/.env.php @@ -1,7 +1,9 @@ __DIR__.'/uploads', - 'SESSION_FILESTORE' => __DIR__.'/app/storage/sessions', + 'FS_REPO' => 'Local', + 'FS_CONF' => [ + 'ENDPOINT' => __DIR__.'/uploads' + ], 'LOG_FILESTORE' => __DIR__.'/app/storage/logs/laravel.log', ]; \ No newline at end of file diff --git a/app/config/session.php b/app/config/session.php index 21d4621675..699f734cdb 100644 --- a/app/config/session.php +++ b/app/config/session.php @@ -46,7 +46,7 @@ | */ - 'files' => Helpers::getEnvVar('SESSION_FILESTORE'), + 'files' => null, /* |-------------------------------------------------------------------------- diff --git a/app/controllers/xapi/DocumentController.php b/app/controllers/xapi/DocumentController.php index f937cddcac..31a984e4ff 100644 --- a/app/controllers/xapi/DocumentController.php +++ b/app/controllers/xapi/DocumentController.php @@ -2,7 +2,7 @@ use Locker\Repository\Document\DocumentRepository as Document; use Carbon\Carbon; use Locker\Helpers\Exceptions as Exceptions; -use Locker\Repository\File\FlyRepository as FileRepository; +use Locker\Repository\File\Factory as FileFactory; abstract class DocumentController extends BaseController { @@ -238,7 +238,7 @@ public function documentResponse($data) { case "text/plain": return \Response::make($document->content, 200, $headers); default: - $stream = (new FileRepository)->stream($document->getFilePath(), []); + $stream = FileFactory::create()->stream($document->getFilePath(), []); return \Response::stream(function () use ($stream) { fpassthru($stream); }, 200, $headers); diff --git a/app/database/migrations/2015_03_31_130813_attachment_rename.php b/app/database/migrations/2015_03_31_130813_attachment_rename.php index 107ea96dc0..afc62891b2 100644 --- a/app/database/migrations/2015_03_31_130813_attachment_rename.php +++ b/app/database/migrations/2015_03_31_130813_attachment_rename.php @@ -7,7 +7,7 @@ class AttachmentRename extends Migration { public function up() { - $uploads = Helpers::getEnvVar('LOCAL_FILESTORE'); + $uploads = Helpers::getEnvVar('FS_CONF')['ENDPOINT']; $LRSs = $this->getDirectores($uploads); // Gets the attachments. diff --git a/app/locker/repository/File/Factory.php b/app/locker/repository/File/Factory.php new file mode 100644 index 0000000000..f48b08d54a --- /dev/null +++ b/app/locker/repository/File/Factory.php @@ -0,0 +1,14 @@ +filesystem = new Filesystem($adapter); } diff --git a/app/locker/repository/File/LocalFlyRepository.php b/app/locker/repository/File/LocalFlyRepository.php new file mode 100644 index 0000000000..8dc30902cd --- /dev/null +++ b/app/locker/repository/File/LocalFlyRepository.php @@ -0,0 +1,11 @@ +constructFileSystem($adapter); + } + +} diff --git a/app/locker/repository/File/RackspaceFlyRepository.php b/app/locker/repository/File/RackspaceFlyRepository.php new file mode 100644 index 0000000000..54db5ed6c9 --- /dev/null +++ b/app/locker/repository/File/RackspaceFlyRepository.php @@ -0,0 +1,19 @@ + $conf['USERNAME'], + 'password' => $conf['PASSWORD'], + ]); + $store = $client->objectStoreService('cloudFiles', $conf['LOCATION']); + $container = $store->getContainer('flysystem'); + $adapter = new RackspaceAdapter($container); + $this->constructFileSystem($adapter); + } + +} diff --git a/app/locker/repository/File/readme.md b/app/locker/repository/File/readme.md new file mode 100644 index 0000000000..21ffd58ba4 --- /dev/null +++ b/app/locker/repository/File/readme.md @@ -0,0 +1,29 @@ +# File Repository +This page documents how to use different storage adapters in Learning Locker. The supported adapters are listed below. + +- [Local](#local) +- [Rackspace](#rackspace) + +### Local +Your .env.local.php file should look something like this. +```php + 'FS_REPO' => 'Local', + 'FS_CONF' => [ + 'ENDPOINT' => __DIR__.'/uploads' + ], +``` + +### Rackspace +Your .env.local.php file will need this line at the top (below ` 'Rackspace', + 'FS_CONF' => [ + 'ENDPOINT' => Rackspace::UK_IDENTITY_ENDPOINT, + 'USERNAME' => 'YOUR USERNAME', + 'PASSWORD' => 'YOUR PASSWORD', + 'LOCATION' => 'LON' + ], +``` diff --git a/app/locker/repository/RepositoryServiceProvider.php b/app/locker/repository/RepositoryServiceProvider.php index b9ac6b1883..7589be40e0 100644 --- a/app/locker/repository/RepositoryServiceProvider.php +++ b/app/locker/repository/RepositoryServiceProvider.php @@ -46,10 +46,6 @@ public function register(){ 'Locker\Repository\Export\Repository', 'Locker\Repository\Export\EloquentRepository' ); - $this->app->bind( - 'Locker\Repository\File\Repository', - 'Locker\Repository\File\FlyRepository' - ); } diff --git a/app/locker/repository/Statement/FileAttacher.php b/app/locker/repository/Statement/FileAttacher.php index afa7ec74f7..6a82b2841c 100644 --- a/app/locker/repository/Statement/FileAttacher.php +++ b/app/locker/repository/Statement/FileAttacher.php @@ -2,7 +2,7 @@ use Locker\Repository\Document\FileTypes as FileTypes; use Locker\Helpers\Helpers as Helpers; use Locker\Helpers\Exceptions as Exceptions; -use Locker\Repository\File\FlyRepository as FileRepository; +use Locker\Repository\File\Factory as FileFactory; class FileAttacher { @@ -26,7 +26,7 @@ public function store(array $attachments, array $hashes, StoreOptions $opts) { ); $filename = $attachment->hash.'.'.$ext; - (new FileRepository)->update($dir.$filename, ['content' => $attachment->content], []); + FileFactory::create()->update($dir.$filename, ['content' => $attachment->content], []); } } @@ -47,7 +47,7 @@ public function index(array $statements, IndexOptions $opts) { return (object) [ 'content_type' => $attachment->contentType, 'hash' => $attachment->sha2, - 'content' => (new FileRepository)->show($dir.$filename, []) + 'content' => FileFactory::create()->show($dir.$filename, []) ]; }, isset($statement->attachments) ? $statement->attachments : [])); } diff --git a/app/models/DocumentAPI.php b/app/models/DocumentAPI.php index 8db30c0fe2..1f88dfe7e8 100644 --- a/app/models/DocumentAPI.php +++ b/app/models/DocumentAPI.php @@ -9,7 +9,7 @@ use Locker\Repository\Document\FileTypes; use Locker\Helpers\Helpers as Helpers; use Locker\Helpers\Exceptions as Exceptions; -use Locker\Repository\File\FlyRepository as FileRepository; +use Locker\Repository\File\Factory as FileFactory; class DocumentAPI extends Eloquent { protected $collection = 'documentapi'; @@ -94,14 +94,14 @@ private function saveDocument($content, $contentType) { $tmp_location = __DIR__.'/../../uploads/tmp'; $content->move($tmp_location, $filename); $data = file_get_contents($tmp_location.'/'.$filename); - (new FileRepository)->update($dir.$filename, ['content' => $data], []); + FileFactory::create()->update($dir.$filename, ['content' => $data], []); unlink($tmp_location.'/'.$filename); } else { $ext = array_search($contentType, FileTypes::getMap()); $filename = time().'_'.mt_rand(0,1000).($ext !== false ? '.'.$ext : ''); - $size = (new FileRepository)->update($dir.$filename, ['content' => $content], []); + $size = FileFactory::create()->update($dir.$filename, ['content' => $content], []); if ($size === false) throw new Exceptions\Exception('There was an issue saving the content'); } diff --git a/app/tests/routes/StatementAttachmentTest.php b/app/tests/routes/StatementAttachmentTest.php index 51d41862d1..ff97696a41 100644 --- a/app/tests/routes/StatementAttachmentTest.php +++ b/app/tests/routes/StatementAttachmentTest.php @@ -120,7 +120,7 @@ private function deleteDirectory($dir) { public function tearDown() { parent::tearDown(); - $dir = Helpers::getEnvVar('LOCAL_FILESTORE').'/'.$this->lrs->_id; + $dir = Helpers::getEnvVar('FS_CONF')['ENDPOINT'].'/'.$this->lrs->_id; $this->deleteDirectory($dir); (new \Statement) ->where('lrs_id', new \MongoId($this->lrs->_id)) diff --git a/composer.json b/composer.json index b73e93ec64..94bfd55410 100644 --- a/composer.json +++ b/composer.json @@ -44,7 +44,8 @@ "bshaffer/oauth2-server-php": "1.7.x", "bshaffer/oauth2-server-httpfoundation-bridge": "1.1.x", "jenssegers/mongodb-session": "~1.0", - "league/flysystem": "~1.0" + "league/flysystem": "~1.0", + "league/flysystem-rackspace": "^1.0" }, "require-dev": { "phpunit/phpunit": "4.1.*" diff --git a/composer.lock b/composer.lock index ad78544c73..2e3cb2ce76 100644 --- a/composer.lock +++ b/composer.lock @@ -1,10 +1,10 @@ { "_readme": [ "This file locks the dependencies of your project to a known state", - "Read more about it at http://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file", + "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#composer-lock-the-lock-file", "This file is @generated automatically" ], - "hash": "ae8e6f219e167ae88fb097cd215acbe8", + "hash": "3737a0e9c0e7630f6bdb771cf54a1ffd", "packages": [ { "name": "andywer/js-localization", @@ -251,12 +251,12 @@ "version": "1.0.2", "source": { "type": "git", - "url": "https://github.com/mtdowling/ClassPreloader.git", + "url": "https://github.com/ClassPreloader/ClassPreloader.git", "reference": "2c9f3bcbab329570c57339895bd11b5dd3b00877" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/mtdowling/ClassPreloader/zipball/2c9f3bcbab329570c57339895bd11b5dd3b00877", + "url": "https://api.github.com/repos/ClassPreloader/ClassPreloader/zipball/2c9f3bcbab329570c57339895bd11b5dd3b00877", "reference": "2c9f3bcbab329570c57339895bd11b5dd3b00877", "shasum": "" }, @@ -433,6 +433,101 @@ ], "time": "2015-06-29 05:42:04" }, + { + "name": "guzzle/guzzle", + "version": "v3.9.3", + "source": { + "type": "git", + "url": "https://github.com/guzzle/guzzle3.git", + "reference": "0645b70d953bc1c067bbc8d5bc53194706b628d9" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/guzzle/guzzle3/zipball/0645b70d953bc1c067bbc8d5bc53194706b628d9", + "reference": "0645b70d953bc1c067bbc8d5bc53194706b628d9", + "shasum": "" + }, + "require": { + "ext-curl": "*", + "php": ">=5.3.3", + "symfony/event-dispatcher": "~2.1" + }, + "replace": { + "guzzle/batch": "self.version", + "guzzle/cache": "self.version", + "guzzle/common": "self.version", + "guzzle/http": "self.version", + "guzzle/inflection": "self.version", + "guzzle/iterator": "self.version", + "guzzle/log": "self.version", + "guzzle/parser": "self.version", + "guzzle/plugin": "self.version", + "guzzle/plugin-async": "self.version", + "guzzle/plugin-backoff": "self.version", + "guzzle/plugin-cache": "self.version", + "guzzle/plugin-cookie": "self.version", + "guzzle/plugin-curlauth": "self.version", + "guzzle/plugin-error-response": "self.version", + "guzzle/plugin-history": "self.version", + "guzzle/plugin-log": "self.version", + "guzzle/plugin-md5": "self.version", + "guzzle/plugin-mock": "self.version", + "guzzle/plugin-oauth": "self.version", + "guzzle/service": "self.version", + "guzzle/stream": "self.version" + }, + "require-dev": { + "doctrine/cache": "~1.3", + "monolog/monolog": "~1.0", + "phpunit/phpunit": "3.7.*", + "psr/log": "~1.0", + "symfony/class-loader": "~2.1", + "zendframework/zend-cache": "2.*,<2.3", + "zendframework/zend-log": "2.*,<2.3" + }, + "suggest": { + "guzzlehttp/guzzle": "Guzzle 5 has moved to a new package name. The package you have installed, Guzzle 3, is deprecated." + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "3.9-dev" + } + }, + "autoload": { + "psr-0": { + "Guzzle": "src/", + "Guzzle\\Tests": "tests/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Michael Dowling", + "email": "mtdowling@gmail.com", + "homepage": "https://github.com/mtdowling" + }, + { + "name": "Guzzle Community", + "homepage": "https://github.com/guzzle/guzzle/contributors" + } + ], + "description": "PHP HTTP client. This library is deprecated in favor of https://packagist.org/packages/guzzlehttp/guzzle", + "homepage": "http://guzzlephp.org/", + "keywords": [ + "client", + "curl", + "framework", + "http", + "http client", + "rest", + "web service" + ], + "time": "2015-03-18 18:23:50" + }, { "name": "ircmaxell/password-compat", "version": "v1.0.4", @@ -822,6 +917,53 @@ ], "time": "2015-07-28 20:41:58" }, + { + "name": "league/flysystem-rackspace", + "version": "1.0.2", + "source": { + "type": "git", + "url": "https://github.com/thephpleague/flysystem-rackspace.git", + "reference": "d60b1b0a270d51b74a17c879de04e615ead84d38" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/thephpleague/flysystem-rackspace/zipball/d60b1b0a270d51b74a17c879de04e615ead84d38", + "reference": "d60b1b0a270d51b74a17c879de04e615ead84d38", + "shasum": "" + }, + "require": { + "league/flysystem": "~1.0", + "php": ">=5.4.0", + "rackspace/php-opencloud": "~1.12" + }, + "require-dev": { + "mockery/mockery": "0.9.*", + "phpunit/phpunit": "~4.0" + }, + "type": "library", + "extra": { + "branch-alias": { + "dev-master": "1.0-dev" + } + }, + "autoload": { + "psr-4": { + "League\\Flysystem\\Rackspace\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Frank de Jonge", + "email": "info@frenky.net" + } + ], + "description": "Flysystem adapter for Rackspace", + "time": "2015-06-11 14:35:30" + }, { "name": "learninglocker/statementfactory", "version": "v0.0.2", @@ -855,6 +997,33 @@ "description": "Learning Locker's classes for xAPI.", "time": "2015-05-13 14:33:50" }, + { + "name": "mikemccabe/json-patch-php", + "version": "0.1.0", + "source": { + "type": "git", + "url": "https://github.com/mikemccabe/json-patch-php.git", + "reference": "b3af30a6aec7f6467c773cd49b2d974a70f7c0d4" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/mikemccabe/json-patch-php/zipball/b3af30a6aec7f6467c773cd49b2d974a70f7c0d4", + "reference": "b3af30a6aec7f6467c773cd49b2d974a70f7c0d4", + "shasum": "" + }, + "type": "library", + "autoload": { + "psr-4": { + "mikemccabe\\JsonPatch\\": "src" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "LGPL-3.0" + ], + "description": "Produce and apply json-patch objects", + "time": "2015-01-05 21:19:54" + }, { "name": "monolog/monolog", "version": "1.16.0", @@ -1310,6 +1479,73 @@ ], "time": "2012-12-21 11:40:51" }, + { + "name": "rackspace/php-opencloud", + "version": "v1.15.0", + "source": { + "type": "git", + "url": "https://github.com/rackspace/php-opencloud.git", + "reference": "6737f9e4580106fa7b690fdad4f8016c99e2f445" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/rackspace/php-opencloud/zipball/6737f9e4580106fa7b690fdad4f8016c99e2f445", + "reference": "6737f9e4580106fa7b690fdad4f8016c99e2f445", + "shasum": "" + }, + "require": { + "guzzle/guzzle": "~3.8", + "mikemccabe/json-patch-php": "~0.1", + "php": ">=5.4", + "psr/log": "~1.0" + }, + "require-dev": { + "apigen/apigen": "~4.0", + "fabpot/php-cs-fixer": "1.0.*@dev", + "jakub-onderka/php-parallel-lint": "0.*", + "phpspec/prophecy": "~1.4", + "phpunit/phpunit": "4.3.*", + "satooshi/php-coveralls": "0.6.*@dev" + }, + "type": "library", + "autoload": { + "psr-0": { + "OpenCloud": [ + "lib/", + "tests/" + ] + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "Apache-2.0" + ], + "authors": [ + { + "name": "Glen Campbell", + "email": "glen.campbell@rackspace.com" + }, + { + "name": "Jamie Hannaford", + "email": "jamie.hannaford@rackspace.com", + "homepage": "https://github.com/jamiehannaford" + }, + { + "name": "Shaunak Kashyap", + "email": "shaunak.kashyap@rackspace.com", + "homepage": "https://github.com/ycombinator" + } + ], + "description": "PHP SDK for Rackspace/OpenStack APIs", + "keywords": [ + "Openstack", + "nova", + "opencloud", + "rackspace", + "swift" + ], + "time": "2015-08-04 08:48:38" + }, { "name": "ramsey/array_column", "version": "1.1.3", From bfa6c94db76233af9edfb84d259ea1081d576ed2 Mon Sep 17 00:00:00 2001 From: Ryan Smith <0ryansmith1994@gmail.com> Date: Wed, 19 Aug 2015 15:41:28 +0100 Subject: [PATCH 07/19] Corrects Rackspace support. --- app/locker/repository/File/RackspaceFlyRepository.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/locker/repository/File/RackspaceFlyRepository.php b/app/locker/repository/File/RackspaceFlyRepository.php index 54db5ed6c9..30e89be4cf 100644 --- a/app/locker/repository/File/RackspaceFlyRepository.php +++ b/app/locker/repository/File/RackspaceFlyRepository.php @@ -2,8 +2,7 @@ use OpenCloud\OpenStack as OpenStack; use League\Flysystem\Rackspace\RackspaceAdapter as RackspaceAdapter; -class LocalFlyRepository implements Repository { - protected $filesystem; +class RackspaceFlyRepository extends FlyRepository { public function __construct(array $conf) { $client = new OpenStack($conf['ENDPOINT'], [ From 191c69127bb72f4887ab1cbc3cd9094f8aaa6b58 Mon Sep 17 00:00:00 2001 From: Ryan Smith <0ryansmith1994@gmail.com> Date: Thu, 20 Aug 2015 11:08:24 +0100 Subject: [PATCH 08/19] Fixes streaming and Rackspace support. --- .env.php | 4 +--- .gitignore | 2 ++ app/controllers/xapi/DocumentController.php | 5 ++++- app/locker/helpers/Helpers.php | 1 + app/locker/repository/File/Factory.php | 5 ++--- app/locker/repository/File/FlyRepository.php | 6 +++--- app/locker/repository/File/LocalFlyRepository.php | 5 +++-- .../repository/File/RackspaceFlyRepository.php | 15 ++++++++------- app/locker/repository/File/readme.md | 12 ++++++++++-- 9 files changed, 34 insertions(+), 21 deletions(-) diff --git a/.env.php b/.env.php index 5de895fd5b..32c19f7204 100644 --- a/.env.php +++ b/.env.php @@ -2,8 +2,6 @@ return [ 'FS_REPO' => 'Local', - 'FS_CONF' => [ - 'ENDPOINT' => __DIR__.'/uploads' - ], + 'FS_ENDPOINT' => __DIR__.'/uploads', 'LOG_FILESTORE' => __DIR__.'/app/storage/logs/laravel.log', ]; \ No newline at end of file diff --git a/.gitignore b/.gitignore index 6d5d112397..e7fbeabd4d 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,8 @@ composer.phar .DS_Store Thumbs.db +.phpintel +.env.*.php /app/config/ht2 /app/config/production diff --git a/app/controllers/xapi/DocumentController.php b/app/controllers/xapi/DocumentController.php index 31a984e4ff..945e92c45a 100644 --- a/app/controllers/xapi/DocumentController.php +++ b/app/controllers/xapi/DocumentController.php @@ -240,7 +240,10 @@ public function documentResponse($data) { default: $stream = FileFactory::create()->stream($document->getFilePath(), []); return \Response::stream(function () use ($stream) { - fpassthru($stream); + while (!feof($stream)) { + echo fread($stream, 8192); flush();ob_flush(); + } + fclose($stream); }, 200, $headers); } } diff --git a/app/locker/helpers/Helpers.php b/app/locker/helpers/Helpers.php index 7d2c663c4b..8eb891bee5 100644 --- a/app/locker/helpers/Helpers.php +++ b/app/locker/helpers/Helpers.php @@ -114,6 +114,7 @@ static function getEnvironment($config, $givenHost) { static function getEnvVar($var) { $value = getenv($var); + \Log::info($var, [json_encode($value)]); if ($value === false) { $defaults = include base_path() . '/.env.php'; $value = $defaults[$var]; diff --git a/app/locker/repository/File/Factory.php b/app/locker/repository/File/Factory.php index f48b08d54a..a79bad4990 100644 --- a/app/locker/repository/File/Factory.php +++ b/app/locker/repository/File/Factory.php @@ -4,11 +4,10 @@ class Factory { public static function create() { $FS_REPO = Helpers::getEnvVar('FS_REPO'); - $FS_CONF = Helpers::getEnvVar('FS_CONF'); switch ($FS_REPO) { - case 'Local': return new LocalFlyRepository($FS_CONF); - case 'Rackspace': return new RackspaceFlyRepositry($FS_CONF); + case 'Rackspace': return new RackspaceFlyRepository(); + default: return new LocalFlyRepository(); } } } \ No newline at end of file diff --git a/app/locker/repository/File/FlyRepository.php b/app/locker/repository/File/FlyRepository.php index a2621a8647..68509bc3e3 100644 --- a/app/locker/repository/File/FlyRepository.php +++ b/app/locker/repository/File/FlyRepository.php @@ -5,8 +5,6 @@ abstract class FlyRepository implements Repository { protected $filesystem; - public function __construct(array $conf) {} - protected function constructFileSystem(AdapterInterface $adapter) { $this->filesystem = new Filesystem($adapter); } @@ -34,7 +32,9 @@ public function update($id, array $data, array $opts) { } public function stream($id, array $opts) { - return $this->filesystem->readStream($id); + $stream = $this->filesystem->readStream($id); + fseek($stream, 0); + return $stream; } } diff --git a/app/locker/repository/File/LocalFlyRepository.php b/app/locker/repository/File/LocalFlyRepository.php index 8dc30902cd..891d2605da 100644 --- a/app/locker/repository/File/LocalFlyRepository.php +++ b/app/locker/repository/File/LocalFlyRepository.php @@ -1,10 +1,11 @@ constructFileSystem($adapter); } diff --git a/app/locker/repository/File/RackspaceFlyRepository.php b/app/locker/repository/File/RackspaceFlyRepository.php index 30e89be4cf..5d2c032c6b 100644 --- a/app/locker/repository/File/RackspaceFlyRepository.php +++ b/app/locker/repository/File/RackspaceFlyRepository.php @@ -1,16 +1,17 @@ $conf['USERNAME'], - 'password' => $conf['PASSWORD'], + public function __construct() { + $client = new Rackspace(Helpers::getEnvVar('FS_ENDPOINT'), [ + 'username' => Helpers::getEnvVar('FS_USERNAME'), + 'apiKey' => Helpers::getEnvVar('FS_API_KEY'), ]); - $store = $client->objectStoreService('cloudFiles', $conf['LOCATION']); - $container = $store->getContainer('flysystem'); + $store = $client->objectStoreService('cloudFiles', Helpers::getEnvVar('FS_REGION')); + $container = $store->getContainer(Helpers::getEnvVar('FS_CONTAINER')); $adapter = new RackspaceAdapter($container); $this->constructFileSystem($adapter); } diff --git a/app/locker/repository/File/readme.md b/app/locker/repository/File/readme.md index 21ffd58ba4..b417a3ee9d 100644 --- a/app/locker/repository/File/readme.md +++ b/app/locker/repository/File/readme.md @@ -15,6 +15,7 @@ Your .env.local.php file should look something like this. ### Rackspace Your .env.local.php file will need this line at the top (below ` [ 'ENDPOINT' => Rackspace::UK_IDENTITY_ENDPOINT, 'USERNAME' => 'YOUR USERNAME', - 'PASSWORD' => 'YOUR PASSWORD', - 'LOCATION' => 'LON' + 'API_KEY' => 'YOUR API KEY', + 'REGION' => 'LON', + 'CONTAINER' => 'YOUR CONTAINER' ], ``` + +### Useful Development Links +http://docs.rackspace.com/sdks/api/php/class-OpenCloud.Rackspace.html +http://flysystem.thephpleague.com/api/ +https://github.com/thephpleague/flysystem-aws-s3-v2/issues/3 +https://github.com/thephpleague/flysystem-rackspace/issues/7 \ No newline at end of file From ab11cf7ecf6937fc634d8f8fbb4c0c54f9c09720 Mon Sep 17 00:00:00 2001 From: Ryan Smith <0ryansmith1994@gmail.com> Date: Thu, 20 Aug 2015 11:09:33 +0100 Subject: [PATCH 09/19] Corrects docs. --- app/locker/repository/File/readme.md | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/app/locker/repository/File/readme.md b/app/locker/repository/File/readme.md index b417a3ee9d..839c3d5899 100644 --- a/app/locker/repository/File/readme.md +++ b/app/locker/repository/File/readme.md @@ -8,9 +8,7 @@ This page documents how to use different storage adapters in Learning Locker. Th Your .env.local.php file should look something like this. ```php 'FS_REPO' => 'Local', - 'FS_CONF' => [ - 'ENDPOINT' => __DIR__.'/uploads' - ], + 'FS_ENDPOINT' => __DIR__.'/uploads', ``` ### Rackspace @@ -21,13 +19,11 @@ Your .env.local.php file will need this line at the top (below ` 'Rackspace', - 'FS_CONF' => [ - 'ENDPOINT' => Rackspace::UK_IDENTITY_ENDPOINT, - 'USERNAME' => 'YOUR USERNAME', - 'API_KEY' => 'YOUR API KEY', - 'REGION' => 'LON', - 'CONTAINER' => 'YOUR CONTAINER' - ], + 'FS_ENDPOINT' => Rackspace::UK_IDENTITY_ENDPOINT, + 'FS_USERNAME' => 'YOUR USERNAME', + 'FS_API_KEY' => 'YOUR API KEY', + 'FS_REGION' => 'LON', + 'FS_CONTAINER' => 'YOUR CONTAINER', ``` ### Useful Development Links From 93f97342082cccf5f6a0e945939814067e26b4ca Mon Sep 17 00:00:00 2001 From: Ryan Smith <0ryansmith1994@gmail.com> Date: Thu, 20 Aug 2015 11:12:11 +0100 Subject: [PATCH 10/19] Updates docs. --- app/locker/repository/File/readme.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/app/locker/repository/File/readme.md b/app/locker/repository/File/readme.md index 839c3d5899..8906cdafdf 100644 --- a/app/locker/repository/File/readme.md +++ b/app/locker/repository/File/readme.md @@ -12,9 +12,10 @@ Your .env.local.php file should look something like this. ``` ### Rackspace -Your .env.local.php file will need this line at the top (below ` Date: Thu, 20 Aug 2015 11:14:54 +0100 Subject: [PATCH 11/19] Improves factory repo name recognition. --- app/locker/repository/File/Factory.php | 1 + 1 file changed, 1 insertion(+) diff --git a/app/locker/repository/File/Factory.php b/app/locker/repository/File/Factory.php index a79bad4990..4c74fed301 100644 --- a/app/locker/repository/File/Factory.php +++ b/app/locker/repository/File/Factory.php @@ -4,6 +4,7 @@ class Factory { public static function create() { $FS_REPO = Helpers::getEnvVar('FS_REPO'); + $FS_REPO = ucfirst(strtolower($FS_REPO)); switch ($FS_REPO) { case 'Rackspace': return new RackspaceFlyRepository(); From a91cee87953220db8c785479700acc8ea2b2005f Mon Sep 17 00:00:00 2001 From: Ryan Smith <0ryansmith1994@gmail.com> Date: Thu, 20 Aug 2015 11:17:54 +0100 Subject: [PATCH 12/19] Corrects FS_ENDPOINT usage. --- app/database/migrations/2015_03_31_130813_attachment_rename.php | 2 +- app/tests/routes/StatementAttachmentTest.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/database/migrations/2015_03_31_130813_attachment_rename.php b/app/database/migrations/2015_03_31_130813_attachment_rename.php index afc62891b2..e68921b241 100644 --- a/app/database/migrations/2015_03_31_130813_attachment_rename.php +++ b/app/database/migrations/2015_03_31_130813_attachment_rename.php @@ -7,7 +7,7 @@ class AttachmentRename extends Migration { public function up() { - $uploads = Helpers::getEnvVar('FS_CONF')['ENDPOINT']; + $uploads = Helpers::getEnvVar('FS_ENDPOINT'); $LRSs = $this->getDirectores($uploads); // Gets the attachments. diff --git a/app/tests/routes/StatementAttachmentTest.php b/app/tests/routes/StatementAttachmentTest.php index ff97696a41..7247f1eca8 100644 --- a/app/tests/routes/StatementAttachmentTest.php +++ b/app/tests/routes/StatementAttachmentTest.php @@ -120,7 +120,7 @@ private function deleteDirectory($dir) { public function tearDown() { parent::tearDown(); - $dir = Helpers::getEnvVar('FS_CONF')['ENDPOINT'].'/'.$this->lrs->_id; + $dir = Helpers::getEnvVar('FS_ENDPOINT').'/'.$this->lrs->_id; $this->deleteDirectory($dir); (new \Statement) ->where('lrs_id', new \MongoId($this->lrs->_id)) From db9d5e0279759a082d689c5802f65c48b30d3232 Mon Sep 17 00:00:00 2001 From: Ryan Smith <0ryansmith1994@gmail.com> Date: Thu, 20 Aug 2015 14:52:10 +0100 Subject: [PATCH 13/19] Streams attachments. --- .../xapi/StatementIndexController.php | 45 +++++++++++++------ app/locker/helpers/Helpers.php | 1 - .../repository/Statement/FileAttacher.php | 2 +- app/tests/routes/OAuthTest.php | 3 +- app/tests/routes/StatementAttachmentTest.php | 2 + 5 files changed, 36 insertions(+), 17 deletions(-) diff --git a/app/controllers/xapi/StatementIndexController.php b/app/controllers/xapi/StatementIndexController.php index fe1cf85e99..fe5eab1fba 100644 --- a/app/controllers/xapi/StatementIndexController.php +++ b/app/controllers/xapi/StatementIndexController.php @@ -49,17 +49,21 @@ public function index($options) { // Defines the content type and body of the response. if ($opts['attachments'] === true) { $content_type = 'multipart/mixed; boundary='.static::BOUNDARY; - $body = $this->makeAttachmentsResult($statements, $count, $opts); + $body = function () use ($statements, $count, $opts) { + return $this->makeAttachmentsResult($statements, $count, $opts); + }; } else { $content_type = 'application/json;'; - $body = $this->makeStatementsResult($statements, $count, $opts); + $body = function () use ($statements, $count, $opts) { + return $this->makeStatementsResult($statements, $count, $opts); + }; } // Creates the response. - return \Response::make($body, 200, [ + return \Response::stream($body, 200, [ 'Content-Type' => $content_type, 'X-Experience-API-Consistent-Through' => Helpers::getCurrentDate() - ]);; + ]); } /** @@ -83,7 +87,7 @@ private function makeStatementsResult(array $statements, $count, array $opts) { 'statements' => $statements ]; - return json_encode($statement_result); + $this->emit(json_encode($statement_result)); } /** @@ -96,24 +100,37 @@ private function makeAttachmentsResult(array $statements, $count, array $opts) { $boundary = static::BOUNDARY; $eol = static::EOL; $content_type = 'multipart/mixed; boundary='.$boundary; - $statement_result = "Content-Type:application/json$eol$eol".$this->makeStatementsResult( + + $this->emit("--$boundary$eol"); + $this->emit("Content-Type:application/json$eol$eol"); + $this->makeStatementsResult( $statements, $count, $opts ); - return "--$boundary$eol".implode( - "$eol--$boundary$eol", - array_merge([$statement_result], array_map(function ($attachment) use ($eol, $boundary) { - return ( + $attachments = $this->statements->getAttachments($statements, $opts); + foreach ($attachments as $attachment) { + $this->emit( + "$eol--$boundary$eol". 'Content-Type:'.$attachment->content_type.$eol. 'Content-Transfer-Encoding:binary'.$eol. 'X-Experience-API-Hash:'.$attachment->hash. - $eol.$eol. - $attachment->content + $eol.$eol ); - }, $this->statements->getAttachments($statements, $opts))) - )."$eol--$boundary--"; + while (!feof($attachment->content)) { + $this->emit(fread($attachment->content, 8192)); + } + fclose($attachment->content); + } + + $this->emit("$eol--$boundary--"); + } + + private function emit($value) { + echo $value; + flush(); + ob_flush(); } private function getMoreLink($count, $limit, $offset) { diff --git a/app/locker/helpers/Helpers.php b/app/locker/helpers/Helpers.php index 8eb891bee5..7d2c663c4b 100644 --- a/app/locker/helpers/Helpers.php +++ b/app/locker/helpers/Helpers.php @@ -114,7 +114,6 @@ static function getEnvironment($config, $givenHost) { static function getEnvVar($var) { $value = getenv($var); - \Log::info($var, [json_encode($value)]); if ($value === false) { $defaults = include base_path() . '/.env.php'; $value = $defaults[$var]; diff --git a/app/locker/repository/Statement/FileAttacher.php b/app/locker/repository/Statement/FileAttacher.php index 6a82b2841c..9ad87a5ebc 100644 --- a/app/locker/repository/Statement/FileAttacher.php +++ b/app/locker/repository/Statement/FileAttacher.php @@ -47,7 +47,7 @@ public function index(array $statements, IndexOptions $opts) { return (object) [ 'content_type' => $attachment->contentType, 'hash' => $attachment->sha2, - 'content' => FileFactory::create()->show($dir.$filename, []) + 'content' => FileFactory::create()->stream($dir.$filename, []) ]; }, isset($statement->attachments) ? $statement->attachments : [])); } diff --git a/app/tests/routes/OAuthTest.php b/app/tests/routes/OAuthTest.php index 0287a20bd1..4842eb81ae 100644 --- a/app/tests/routes/OAuthTest.php +++ b/app/tests/routes/OAuthTest.php @@ -61,8 +61,9 @@ public function testAccessTokenViaClient() { public function testAccessViaToken() { $token_response = $this->requestToken($this->ll_client->api['basic_key'], $this->ll_client->api['basic_secret']); $token = json_decode($token_response->getContent())->access_token; + ob_start(); $response = $this->requestApi($token); - $data = $response->getContent(); + $data = ob_get_flush(); // Checks result object. $this->assertEquals('{"more":"","statements":[]}', $data); diff --git a/app/tests/routes/StatementAttachmentTest.php b/app/tests/routes/StatementAttachmentTest.php index 7247f1eca8..5180beab8e 100644 --- a/app/tests/routes/StatementAttachmentTest.php +++ b/app/tests/routes/StatementAttachmentTest.php @@ -85,6 +85,7 @@ public function testIndexAttachments() { ->where('statement.id', $this->statement_id) ->first()->statement; + ob_start(); $response = $this->requestStatements('GET', [ 'attachments' => true ]); @@ -94,6 +95,7 @@ public function testIndexAttachments() { // Checks that the content is correct. $actual_content = str_replace("\r", "", $response->getContent()); + $actual_content = str_replace("\r", "", ob_get_flush()); $expected_content = str_replace("\r", "", $this->generateContent(json_encode([ 'more' => '', 'statements' => [$statement] From ce0b8c6e825eeddee9ca042dc8bce8f10675a193 Mon Sep 17 00:00:00 2001 From: Ryan Smith <0ryansmith1994@gmail.com> Date: Thu, 20 Aug 2015 15:08:10 +0100 Subject: [PATCH 14/19] Cleans the buffer instead of flushing to stop echo calls in the response going to stdout. --- app/tests/routes/OAuthTest.php | 2 +- app/tests/routes/StatementAttachmentTest.php | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/app/tests/routes/OAuthTest.php b/app/tests/routes/OAuthTest.php index 4842eb81ae..ccf7017b9a 100644 --- a/app/tests/routes/OAuthTest.php +++ b/app/tests/routes/OAuthTest.php @@ -63,7 +63,7 @@ public function testAccessViaToken() { $token = json_decode($token_response->getContent())->access_token; ob_start(); $response = $this->requestApi($token); - $data = ob_get_flush(); + $data = ob_get_clean(); // Checks result object. $this->assertEquals('{"more":"","statements":[]}', $data); diff --git a/app/tests/routes/StatementAttachmentTest.php b/app/tests/routes/StatementAttachmentTest.php index 5180beab8e..87bbb48c8c 100644 --- a/app/tests/routes/StatementAttachmentTest.php +++ b/app/tests/routes/StatementAttachmentTest.php @@ -94,8 +94,7 @@ public function testIndexAttachments() { $attachment['content'] .= PHP_EOL; // Checks that the content is correct. - $actual_content = str_replace("\r", "", $response->getContent()); - $actual_content = str_replace("\r", "", ob_get_flush()); + $actual_content = str_replace("\r", "", ob_get_clean()); $expected_content = str_replace("\r", "", $this->generateContent(json_encode([ 'more' => '', 'statements' => [$statement] From 05b2b9e2e8062c9df6a8908889f1befc022d77b6 Mon Sep 17 00:00:00 2001 From: Ryan Smith <0ryansmith1994@gmail.com> Date: Thu, 20 Aug 2015 17:07:55 +0100 Subject: [PATCH 15/19] Adds command to migrate file repos. --- .env.php | 2 +- app/locker/repository/File/Factory.php | 24 +++++++++++++++---- app/locker/repository/File/FlyRepository.php | 8 ++++++- .../repository/File/LocalFlyRepository.php | 5 ++-- .../File/RackspaceFlyRepository.php | 13 +++++----- app/locker/repository/File/readme.md | 12 +++++----- app/start/artisan.php | 1 + 7 files changed, 42 insertions(+), 23 deletions(-) diff --git a/.env.php b/.env.php index 32c19f7204..c64de62de6 100644 --- a/.env.php +++ b/.env.php @@ -2,6 +2,6 @@ return [ 'FS_REPO' => 'Local', - 'FS_ENDPOINT' => __DIR__.'/uploads', + 'FS_LOCAL_ENDPOINT' => __DIR__.'/uploads', 'LOG_FILESTORE' => __DIR__.'/app/storage/logs/laravel.log', ]; \ No newline at end of file diff --git a/app/locker/repository/File/Factory.php b/app/locker/repository/File/Factory.php index 4c74fed301..4987e72cfc 100644 --- a/app/locker/repository/File/Factory.php +++ b/app/locker/repository/File/Factory.php @@ -3,12 +3,26 @@ class Factory { public static function create() { - $FS_REPO = Helpers::getEnvVar('FS_REPO'); - $FS_REPO = ucfirst(strtolower($FS_REPO)); + $repo = Helpers::getEnvVar('FS_REPO'); - switch ($FS_REPO) { - case 'Rackspace': return new RackspaceFlyRepository(); - default: return new LocalFlyRepository(); + return $this->createRepo($repo); + } + + public static function createRepo($repo) { + $repos = [ + 'Local' => 'LocalFlyRepository', + 'Rackspace' => 'RackspaceFlyRepository', + ]; + $repo = ucfirst(strtolower($repo)); + $conf = function ($var) { + return Helpers::getEnvVar($var); + }; + + if (isset($repos[$repo])) { + $selected_repo = 'Locker\Repository\File\\'.$repos[$repo]; + return new $selected_repo($conf); + } else { + throw new \Exception('Valid `FS_REPO` not specified in ".env.'.\App::environment().'.php". Valid values include: "'.implode('", "', array_keys($repos)).'". You provided "'.$repo.'".'); } } } \ No newline at end of file diff --git a/app/locker/repository/File/FlyRepository.php b/app/locker/repository/File/FlyRepository.php index 68509bc3e3..15dd502430 100644 --- a/app/locker/repository/File/FlyRepository.php +++ b/app/locker/repository/File/FlyRepository.php @@ -5,12 +5,14 @@ abstract class FlyRepository implements Repository { protected $filesystem; + public function __construct(Callable $conf) {} + protected function constructFileSystem(AdapterInterface $adapter) { $this->filesystem = new Filesystem($adapter); } public function index(array $opts) { - throw new \BadMethodCallException(); + return $this->filesystem->listContents('.', true); } public function show($id, array $opts) { @@ -37,4 +39,8 @@ public function stream($id, array $opts) { return $stream; } + public function has($id, array $opts) { + return $this->filesystem->has($id); + } + } diff --git a/app/locker/repository/File/LocalFlyRepository.php b/app/locker/repository/File/LocalFlyRepository.php index 891d2605da..af3c7d9575 100644 --- a/app/locker/repository/File/LocalFlyRepository.php +++ b/app/locker/repository/File/LocalFlyRepository.php @@ -1,11 +1,10 @@ constructFileSystem($adapter); } diff --git a/app/locker/repository/File/RackspaceFlyRepository.php b/app/locker/repository/File/RackspaceFlyRepository.php index 5d2c032c6b..1a9d681639 100644 --- a/app/locker/repository/File/RackspaceFlyRepository.php +++ b/app/locker/repository/File/RackspaceFlyRepository.php @@ -1,17 +1,16 @@ Helpers::getEnvVar('FS_USERNAME'), - 'apiKey' => Helpers::getEnvVar('FS_API_KEY'), + public function __construct(Callable $conf) { + $client = new Rackspace($conf('FS_RACK_ENDPOINT'), [ + 'username' => $conf('FS_RACK_USERNAME'), + 'apiKey' => $conf('FS_RACK_API_KEY'), ]); - $store = $client->objectStoreService('cloudFiles', Helpers::getEnvVar('FS_REGION')); - $container = $store->getContainer(Helpers::getEnvVar('FS_CONTAINER')); + $store = $client->objectStoreService('cloudFiles', $conf('FS_RACK_REGION')); + $container = $store->getContainer($conf('FS_RACK_CONTAINER')); $adapter = new RackspaceAdapter($container); $this->constructFileSystem($adapter); } diff --git a/app/locker/repository/File/readme.md b/app/locker/repository/File/readme.md index 8906cdafdf..c2588aee14 100644 --- a/app/locker/repository/File/readme.md +++ b/app/locker/repository/File/readme.md @@ -8,7 +8,7 @@ This page documents how to use different storage adapters in Learning Locker. Th Your .env.local.php file should look something like this. ```php 'FS_REPO' => 'Local', - 'FS_ENDPOINT' => __DIR__.'/uploads', + 'FS_LOCAL_ENDPOINT' => __DIR__.'/uploads', ``` ### Rackspace @@ -20,11 +20,11 @@ use OpenCloud\Rackspace as Rackspace; Your .env.local.php file should look something like this. ```php 'FS_REPO' => 'Rackspace', - 'FS_ENDPOINT' => Rackspace::UK_IDENTITY_ENDPOINT, - 'FS_USERNAME' => 'YOUR USERNAME', - 'FS_API_KEY' => 'YOUR API KEY', - 'FS_REGION' => 'LON', - 'FS_CONTAINER' => 'YOUR CONTAINER', + 'FS_RACK_ENDPOINT' => Rackspace::UK_IDENTITY_ENDPOINT, + 'FS_RACK_USERNAME' => 'YOUR USERNAME', + 'FS_RACK_API_KEY' => 'YOUR API KEY', + 'FS_RACK_REGION' => 'LON', + 'FS_RACK_CONTAINER' => 'YOUR CONTAINER', ``` ### Useful Development Links diff --git a/app/start/artisan.php b/app/start/artisan.php index d8ef9049d2..5df5f86041 100644 --- a/app/start/artisan.php +++ b/app/start/artisan.php @@ -14,3 +14,4 @@ Artisan::add(new ConvertTimestamp); Artisan::add(new ReportMigrateCommand); Artisan::add(new StatementMigrateCommand); +Artisan::add(new FileRepositoryCommand); From 999db06f6bb45051af9fd3f8ea53feaef98b8640 Mon Sep 17 00:00:00 2001 From: Ryan Smith <0ryansmith1994@gmail.com> Date: Thu, 20 Aug 2015 17:11:29 +0100 Subject: [PATCH 16/19] Commits the FileRepositoryCommand that I forgot to add in the last commit. --- app/commands/FileRepositoryCommand.php | 43 ++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 app/commands/FileRepositoryCommand.php diff --git a/app/commands/FileRepositoryCommand.php b/app/commands/FileRepositoryCommand.php new file mode 100644 index 0000000000..17a692489f --- /dev/null +++ b/app/commands/FileRepositoryCommand.php @@ -0,0 +1,43 @@ +option('from'); + $to_var = $this->argument('to'); + $from_repo = FileFactory::createRepo($from_var); + $to_repo = FileFactory::createRepo($to_var); + $files = $from_repo->index([]); + + $count = 0; + foreach ($files as $file) { + $path = $file['path']; + if ($file['type'] === 'file' && !$to_repo->has($path, [])) { + $count += 1; + $to_repo->update($path, ['content' => $from_repo->show($path, [])], []); + echo "Migrated '{$path}' from '{$from_var}' to '{$to_var}'.".PHP_EOL; + } + } + + echo "Migrated $count files and ignored ".(count($files) - $count).".".PHP_EOL; + } + + protected function getArguments() { + return [ + ['to', InputArgument::REQUIRED, 'The repository to migrate to.'] + ]; + } + + protected function getOptions() { + return [ + ['from', 'f', InputOption::VALUE_OPTIONAL, 'The repository to migrate from.', 'Local'], + ]; + } + +} From 9f2e93ba53c5ac658a056024bfc305f1265a03c4 Mon Sep 17 00:00:00 2001 From: Ryan Smith <0ryansmith1994@gmail.com> Date: Thu, 20 Aug 2015 17:16:56 +0100 Subject: [PATCH 17/19] Fixes tests. --- app/database/migrations/2015_03_31_130813_attachment_rename.php | 2 +- app/locker/repository/File/Factory.php | 2 +- app/tests/routes/StatementAttachmentTest.php | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/database/migrations/2015_03_31_130813_attachment_rename.php b/app/database/migrations/2015_03_31_130813_attachment_rename.php index e68921b241..fc65edcebe 100644 --- a/app/database/migrations/2015_03_31_130813_attachment_rename.php +++ b/app/database/migrations/2015_03_31_130813_attachment_rename.php @@ -7,7 +7,7 @@ class AttachmentRename extends Migration { public function up() { - $uploads = Helpers::getEnvVar('FS_ENDPOINT'); + $uploads = Helpers::getEnvVar('FS_LOCAL_ENDPOINT'); $LRSs = $this->getDirectores($uploads); // Gets the attachments. diff --git a/app/locker/repository/File/Factory.php b/app/locker/repository/File/Factory.php index 4987e72cfc..175273155d 100644 --- a/app/locker/repository/File/Factory.php +++ b/app/locker/repository/File/Factory.php @@ -5,7 +5,7 @@ class Factory { public static function create() { $repo = Helpers::getEnvVar('FS_REPO'); - return $this->createRepo($repo); + return static::createRepo($repo); } public static function createRepo($repo) { diff --git a/app/tests/routes/StatementAttachmentTest.php b/app/tests/routes/StatementAttachmentTest.php index 87bbb48c8c..506cb45499 100644 --- a/app/tests/routes/StatementAttachmentTest.php +++ b/app/tests/routes/StatementAttachmentTest.php @@ -121,7 +121,7 @@ private function deleteDirectory($dir) { public function tearDown() { parent::tearDown(); - $dir = Helpers::getEnvVar('FS_ENDPOINT').'/'.$this->lrs->_id; + $dir = Helpers::getEnvVar('FS_LOCAL_ENDPOINT').'/'.$this->lrs->_id; $this->deleteDirectory($dir); (new \Statement) ->where('lrs_id', new \MongoId($this->lrs->_id)) From dd5b127eab4444b8964fa8f78ae733dbba3f453a Mon Sep 17 00:00:00 2001 From: Ryan Smith <0ryansmith1994@gmail.com> Date: Fri, 21 Aug 2015 10:06:46 +0100 Subject: [PATCH 18/19] Updates documentation with migration command. --- app/locker/repository/File/readme.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/app/locker/repository/File/readme.md b/app/locker/repository/File/readme.md index c2588aee14..fc55979d7c 100644 --- a/app/locker/repository/File/readme.md +++ b/app/locker/repository/File/readme.md @@ -11,6 +11,12 @@ Your .env.local.php file should look something like this. 'FS_LOCAL_ENDPOINT' => __DIR__.'/uploads', ``` +If you're migrating from another file repository (i.e. "Rackspace"), you'll need to run the command below. + +```sh +php artisan ll:file-repo Local -f Rackspace +``` + ### Rackspace Your .env.local.php file will need to include `OpenCloud\Rackspace` (below ` 'YOUR CONTAINER', ``` +If you're migrating from another file repository (i.e. "Local"), you'll need to run the command below. + +```sh +php artisan ll:file-repo Rackspace -f Local +``` + ### Useful Development Links - http://docs.rackspace.com/sdks/api/php/class-OpenCloud.Rackspace.html - http://flysystem.thephpleague.com/api/ From cf2d24f3cef66bf783ba48937fc93179b6bc305b Mon Sep 17 00:00:00 2001 From: Ryan Smith <0ryansmith1994@gmail.com> Date: Fri, 21 Aug 2015 10:08:20 +0100 Subject: [PATCH 19/19] Corrects language of code snippet. --- app/locker/repository/File/readme.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/locker/repository/File/readme.md b/app/locker/repository/File/readme.md index fc55979d7c..04bbc82473 100644 --- a/app/locker/repository/File/readme.md +++ b/app/locker/repository/File/readme.md @@ -13,7 +13,7 @@ Your .env.local.php file should look something like this. If you're migrating from another file repository (i.e. "Rackspace"), you'll need to run the command below. -```sh +```shell php artisan ll:file-repo Local -f Rackspace ``` @@ -35,7 +35,7 @@ Your .env.local.php file should look something like this. If you're migrating from another file repository (i.e. "Local"), you'll need to run the command below. -```sh +```shell php artisan ll:file-repo Rackspace -f Local ```