From 1397e4aa77fbd137c1a9c48fdbdd73a29c5df5c1 Mon Sep 17 00:00:00 2001 From: Tomasz Grobelny Date: Sun, 6 Jan 2019 20:54:52 +0100 Subject: [PATCH 1/5] Write checksum to filecache table Signed-off-by: Tomasz Grobelny --- apps/dav/lib/Connector/Sabre/File.php | 4 ++-- lib/private/Files/Cache/Scanner.php | 5 +++-- lib/private/Files/View.php | 3 +++ 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/apps/dav/lib/Connector/Sabre/File.php b/apps/dav/lib/Connector/Sabre/File.php index f948f0f552d31..39d0820cb3862 100644 --- a/apps/dav/lib/Connector/Sabre/File.php +++ b/apps/dav/lib/Connector/Sabre/File.php @@ -267,7 +267,7 @@ public function put($data) { $checksum = trim($this->request->server['HTTP_OC_CHECKSUM']); $this->fileView->putFileInfo($this->path, ['checksum' => $checksum]); $this->refreshInfo(); - } else if ($this->getChecksum() !== null && $this->getChecksum() !== '') { + } else { $this->fileView->putFileInfo($this->path, ['checksum' => '']); $this->refreshInfo(); } @@ -532,7 +532,7 @@ private function createFileChunked($data) { if (isset($this->request->server['HTTP_OC_CHECKSUM'])) { $checksum = trim($this->request->server['HTTP_OC_CHECKSUM']); $this->fileView->putFileInfo($targetPath, ['checksum' => $checksum]); - } else if ($info->getChecksum() !== null && $info->getChecksum() !== '') { + } else { $this->fileView->putFileInfo($this->path, ['checksum' => '']); } diff --git a/lib/private/Files/Cache/Scanner.php b/lib/private/Files/Cache/Scanner.php index ca9a0b794f99f..9bcd923676e9d 100644 --- a/lib/private/Files/Cache/Scanner.php +++ b/lib/private/Files/Cache/Scanner.php @@ -190,6 +190,9 @@ public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = /** @var CacheEntry $cacheData */ $cacheData = $this->cache->get($file); } + if (empty($cacheData) || $cacheData['mtime'] != $data['mtime'] || $cacheData['checksum']==='') { + $data['checksum'] = md5_file($this->storage->getLocalFile($file)); + } if ($cacheData and $reuseExisting and isset($cacheData['fileid'])) { // prevent empty etag if (empty($cacheData['etag'])) { @@ -216,8 +219,6 @@ public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = $fileId = -1; } if (!empty($newData)) { - // Reset the checksum if the data has changed - $newData['checksum'] = ''; $data['fileid'] = $this->addToCache($file, $newData, $fileId); } if (isset($cacheData['size'])) { diff --git a/lib/private/Files/View.php b/lib/private/Files/View.php index 21df67cf55742..69338cfe2486f 100644 --- a/lib/private/Files/View.php +++ b/lib/private/Files/View.php @@ -1552,6 +1552,9 @@ public function putFileInfo($path, $data) { */ list($storage, $internalPath) = Filesystem::resolvePath($path); if ($storage) { + if ($data['checksum'] == '') { + $data['checksum'] = md5_file($storage->getLocalFile($internalPath)); + } $cache = $storage->getCache($path); if (!$cache->inCache($internalPath)) { From d3c5f2b5d2518067b2c9f6089786bb18fd5b0586 Mon Sep 17 00:00:00 2001 From: Tomasz Grobelny Date: Thu, 10 Jan 2019 21:51:29 +0100 Subject: [PATCH 2/5] Calculate checksum during upload Signed-off-by: Tomasz Grobelny --- apps/dav/lib/Connector/Sabre/File.php | 37 ++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/apps/dav/lib/Connector/Sabre/File.php b/apps/dav/lib/Connector/Sabre/File.php index 39d0820cb3862..af3abe1bcf715 100644 --- a/apps/dav/lib/Connector/Sabre/File.php +++ b/apps/dav/lib/Connector/Sabre/File.php @@ -65,6 +65,28 @@ use Sabre\DAV\IFile; use Sabre\DAV\Exception\NotFound; +class md5sum_filter extends \php_user_filter +{ + function onCreate () { + $this->params->ctx = hash_init('md5'); + return TRUE; + } + + function onClose () { + return TRUE; + } + + function filter ($in, $out, &$consumed, $closing) + { + while ($bucket = stream_bucket_make_writeable($in)) { + hash_update($this->params->ctx, $bucket->data); + $consumed += $bucket->datalen; + stream_bucket_append($out, $bucket); + } + return PSFS_PASS_ON; + } +} + class File extends Node implements IFile { protected $request; @@ -116,6 +138,11 @@ public function __construct(View $view, FileInfo $info, IManager $shareManager = * @return string|null */ public function put($data) { + + stream_filter_register('md5sum', 'OCA\\DAV\\Connector\\Sabre\\md5sum_filter'); + $obj = new \stdClass(); + $md5_filter = stream_filter_append($data, 'md5sum', STREAM_FILTER_READ, $obj); + try { $exists = $this->fileView->file_exists($this->path); if ($this->info && $exists && !$this->info->isUpdateable()) { @@ -261,16 +288,14 @@ public function put($data) { $this->emitPostHooks($exists); } - $this->refreshInfo(); - + stream_filter_remove($md5_filter); if (isset($this->request->server['HTTP_OC_CHECKSUM'])) { $checksum = trim($this->request->server['HTTP_OC_CHECKSUM']); - $this->fileView->putFileInfo($this->path, ['checksum' => $checksum]); - $this->refreshInfo(); } else { - $this->fileView->putFileInfo($this->path, ['checksum' => '']); - $this->refreshInfo(); + $checksum = hash_final($obj->ctx); } + $this->fileView->putFileInfo($this->path, ['checksum' => $checksum]); + $this->refreshInfo(); } catch (StorageNotAvailableException $e) { throw new ServiceUnavailable("Failed to check file size: " . $e->getMessage()); From 1207f1668d4e483c9351b1ef3ad503b3166f6065 Mon Sep 17 00:00:00 2001 From: Tomasz Grobelny Date: Sat, 12 Jan 2019 21:54:22 +0100 Subject: [PATCH 3/5] Tidy up md5 calculating code Signed-off-by: Tomasz Grobelny --- apps/dav/lib/Connector/Sabre/File.php | 30 ++-------------- lib/private/Files/Cache/Scanner.php | 4 +-- lib/private/Files/Utils/Scanner.php | 2 +- lib/private/Files/View.php | 13 +++++-- lib/private/StreamHashFilter.php | 52 +++++++++++++++++++++++++++ lib/public/Files/Cache/IScanner.php | 1 + 6 files changed, 68 insertions(+), 34 deletions(-) create mode 100644 lib/private/StreamHashFilter.php diff --git a/apps/dav/lib/Connector/Sabre/File.php b/apps/dav/lib/Connector/Sabre/File.php index af3abe1bcf715..5271c63945e72 100644 --- a/apps/dav/lib/Connector/Sabre/File.php +++ b/apps/dav/lib/Connector/Sabre/File.php @@ -65,28 +65,6 @@ use Sabre\DAV\IFile; use Sabre\DAV\Exception\NotFound; -class md5sum_filter extends \php_user_filter -{ - function onCreate () { - $this->params->ctx = hash_init('md5'); - return TRUE; - } - - function onClose () { - return TRUE; - } - - function filter ($in, $out, &$consumed, $closing) - { - while ($bucket = stream_bucket_make_writeable($in)) { - hash_update($this->params->ctx, $bucket->data); - $consumed += $bucket->datalen; - stream_bucket_append($out, $bucket); - } - return PSFS_PASS_ON; - } -} - class File extends Node implements IFile { protected $request; @@ -139,10 +117,7 @@ public function __construct(View $view, FileInfo $info, IManager $shareManager = */ public function put($data) { - stream_filter_register('md5sum', 'OCA\\DAV\\Connector\\Sabre\\md5sum_filter'); - $obj = new \stdClass(); - $md5_filter = stream_filter_append($data, 'md5sum', STREAM_FILTER_READ, $obj); - + $contextObject = \OC\StreamHashFilter::getContextObject($data); try { $exists = $this->fileView->file_exists($this->path); if ($this->info && $exists && !$this->info->isUpdateable()) { @@ -288,11 +263,10 @@ public function put($data) { $this->emitPostHooks($exists); } - stream_filter_remove($md5_filter); if (isset($this->request->server['HTTP_OC_CHECKSUM'])) { $checksum = trim($this->request->server['HTTP_OC_CHECKSUM']); } else { - $checksum = hash_final($obj->ctx); + $checksum = hash_final($contextObject->hashContext); } $this->fileView->putFileInfo($this->path, ['checksum' => $checksum]); $this->refreshInfo(); diff --git a/lib/private/Files/Cache/Scanner.php b/lib/private/Files/Cache/Scanner.php index 9bcd923676e9d..db2c66e42c272 100644 --- a/lib/private/Files/Cache/Scanner.php +++ b/lib/private/Files/Cache/Scanner.php @@ -190,8 +190,8 @@ public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = /** @var CacheEntry $cacheData */ $cacheData = $this->cache->get($file); } - if (empty($cacheData) || $cacheData['mtime'] != $data['mtime'] || $cacheData['checksum']==='') { - $data['checksum'] = md5_file($this->storage->getLocalFile($file)); + if (($reuseExisting & self::RECALCULATE_CHECKSUM_IF_EMPTY) && empty($cacheData['checksum'])) { + $data['checksum'] = $this->storage->hash('md5', $file); } if ($cacheData and $reuseExisting and isset($cacheData['fileid'])) { // prevent empty etag diff --git a/lib/private/Files/Utils/Scanner.php b/lib/private/Files/Utils/Scanner.php index 28921973fcf22..2ff619384accf 100644 --- a/lib/private/Files/Utils/Scanner.php +++ b/lib/private/Files/Utils/Scanner.php @@ -242,7 +242,7 @@ public function scan($dir = '', $recursive = \OC\Files\Cache\Scanner::SCAN_RECUR try { $propagator = $storage->getPropagator(); $propagator->beginBatch(); - $scanner->scan($relativePath, $recursive, \OC\Files\Cache\Scanner::REUSE_ETAG | \OC\Files\Cache\Scanner::REUSE_SIZE); + $scanner->scan($relativePath, $recursive, \OC\Files\Cache\Scanner::REUSE_ETAG | \OC\Files\Cache\Scanner::REUSE_SIZE | \OC\Files\Cache\Scanner::RECALCULATE_CHECKSUM_IF_EMPTY); $cache = $storage->getCache(); if ($cache instanceof Cache) { // only re-calculate for the root folder we scanned, anything below that is taken care of by the scanner diff --git a/lib/private/Files/View.php b/lib/private/Files/View.php index 69338cfe2486f..18f68bddad3df 100644 --- a/lib/private/Files/View.php +++ b/lib/private/Files/View.php @@ -637,6 +637,12 @@ protected function emit_file_hooks_post($exists, $path) { * @throws \Exception */ public function file_put_contents($path, $data) { + if (is_string($data)) { + $stream = fopen('php://memory','r+'); + fwrite($stream, $data); + rewind($stream); + $data = $stream; + } if (is_resource($data)) { //not having to deal with streams in file_put_contents makes life easier $absolutePath = Filesystem::normalizePath($this->getAbsolutePath($path)); if (Filesystem::isValidPath($path) @@ -660,11 +666,13 @@ public function file_put_contents($path, $data) { /** @var \OC\Files\Storage\Storage $storage */ list($storage, $internalPath) = $this->resolvePath($path); + $contextObject = \OC\StreamHashFilter::getContextObject($data); $target = $storage->fopen($internalPath, 'w'); if ($target) { list (, $result) = \OC_Helper::streamCopy($data, $target); fclose($target); fclose($data); + $this->putFileInfo($path, ['checksum' => hash_final($contextObject->hashContext)]); $this->writeUpdate($storage, $internalPath); @@ -683,8 +691,7 @@ public function file_put_contents($path, $data) { return false; } } else { - $hooks = $this->file_exists($path) ? array('update', 'write') : array('create', 'write'); - return $this->basicOperation('file_put_contents', $path, $hooks, $data); + throw new \Exception('$data is neither resource nor string'); } } @@ -1553,7 +1560,7 @@ public function putFileInfo($path, $data) { list($storage, $internalPath) = Filesystem::resolvePath($path); if ($storage) { if ($data['checksum'] == '') { - $data['checksum'] = md5_file($storage->getLocalFile($internalPath)); + $data['checksum'] = $storage->hash('md5', $internalPath); } $cache = $storage->getCache($path); diff --git a/lib/private/StreamHashFilter.php b/lib/private/StreamHashFilter.php new file mode 100644 index 0000000000000..961450de4391c --- /dev/null +++ b/lib/private/StreamHashFilter.php @@ -0,0 +1,52 @@ + + * + * @author Tomasz Grobelny + * + * @license AGPL-3.0 + * + * This code is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License, version 3, + * as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License, version 3, + * along with this program. If not, see + * + */ + +namespace OC; + +class StreamHashFilter extends \php_user_filter +{ + static function getContextObject($stream) { + stream_filter_register('hash.md5', 'OC\\StreamHashFilter'); + $obj = new \stdClass(); + $md5_filter = stream_filter_append($stream, 'hash.md5', STREAM_FILTER_ALL, $obj); + return $obj; + } + + function onCreate () { + $this->params->hashContext = hash_init('md5'); + return TRUE; + } + + function onClose () { + return TRUE; + } + + function filter ($in, $out, &$consumed, $closing) + { + while ($bucket = stream_bucket_make_writeable($in)) { + hash_update($this->params->hashContext, $bucket->data); + $consumed += $bucket->datalen; + stream_bucket_append($out, $bucket); + } + return PSFS_PASS_ON; + } +} diff --git a/lib/public/Files/Cache/IScanner.php b/lib/public/Files/Cache/IScanner.php index 8aa4dc04aa90a..71e40c9e037ad 100644 --- a/lib/public/Files/Cache/IScanner.php +++ b/lib/public/Files/Cache/IScanner.php @@ -35,6 +35,7 @@ interface IScanner { const REUSE_NONE = 0; const REUSE_ETAG = 1; const REUSE_SIZE = 2; + const RECALCULATE_CHECKSUM_IF_EMPTY = 4; /** * scan a single file and store it in the cache From e3a39dd0488e9db26d50fab7fd704c12101d2aa4 Mon Sep 17 00:00:00 2001 From: Tomasz Grobelny Date: Sat, 12 Jan 2019 22:36:50 +0100 Subject: [PATCH 4/5] Add hash prefix Signed-off-by: Tomasz Grobelny --- apps/dav/lib/Connector/Sabre/File.php | 2 +- lib/private/Files/Cache/Scanner.php | 2 +- lib/private/Files/View.php | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/dav/lib/Connector/Sabre/File.php b/apps/dav/lib/Connector/Sabre/File.php index 5271c63945e72..7df8d6d9d6416 100644 --- a/apps/dav/lib/Connector/Sabre/File.php +++ b/apps/dav/lib/Connector/Sabre/File.php @@ -268,7 +268,7 @@ public function put($data) { } else { $checksum = hash_final($contextObject->hashContext); } - $this->fileView->putFileInfo($this->path, ['checksum' => $checksum]); + $this->fileView->putFileInfo($this->path, ['checksum' => 'md5:' . $checksum]); $this->refreshInfo(); } catch (StorageNotAvailableException $e) { diff --git a/lib/private/Files/Cache/Scanner.php b/lib/private/Files/Cache/Scanner.php index db2c66e42c272..57c291bee88bc 100644 --- a/lib/private/Files/Cache/Scanner.php +++ b/lib/private/Files/Cache/Scanner.php @@ -191,7 +191,7 @@ public function scanFile($file, $reuseExisting = 0, $parentId = -1, $cacheData = $cacheData = $this->cache->get($file); } if (($reuseExisting & self::RECALCULATE_CHECKSUM_IF_EMPTY) && empty($cacheData['checksum'])) { - $data['checksum'] = $this->storage->hash('md5', $file); + $data['checksum'] = 'md5:' . $this->storage->hash('md5', $file); } if ($cacheData and $reuseExisting and isset($cacheData['fileid'])) { // prevent empty etag diff --git a/lib/private/Files/View.php b/lib/private/Files/View.php index 18f68bddad3df..6d56268e399e2 100644 --- a/lib/private/Files/View.php +++ b/lib/private/Files/View.php @@ -672,7 +672,7 @@ public function file_put_contents($path, $data) { list (, $result) = \OC_Helper::streamCopy($data, $target); fclose($target); fclose($data); - $this->putFileInfo($path, ['checksum' => hash_final($contextObject->hashContext)]); + $this->putFileInfo($path, ['checksum' => 'md5:' . hash_final($contextObject->hashContext)]); $this->writeUpdate($storage, $internalPath); @@ -1560,7 +1560,7 @@ public function putFileInfo($path, $data) { list($storage, $internalPath) = Filesystem::resolvePath($path); if ($storage) { if ($data['checksum'] == '') { - $data['checksum'] = $storage->hash('md5', $internalPath); + $data['checksum'] = 'md5' . $storage->hash('md5', $internalPath); } $cache = $storage->getCache($path); From 4fe0d30a35b613cd04796bdd15bc228076900122 Mon Sep 17 00:00:00 2001 From: Tomasz Grobelny Date: Sat, 12 Jan 2019 23:40:48 +0100 Subject: [PATCH 5/5] Update autoload files Signed-off-by: Tomasz Grobelny --- lib/composer/composer/autoload_classmap.php | 1 + lib/composer/composer/autoload_static.php | 1 + 2 files changed, 2 insertions(+) diff --git a/lib/composer/composer/autoload_classmap.php b/lib/composer/composer/autoload_classmap.php index 299105448952f..9e232323d5740 100644 --- a/lib/composer/composer/autoload_classmap.php +++ b/lib/composer/composer/autoload_classmap.php @@ -1074,6 +1074,7 @@ 'OC\\Share\\Helper' => $baseDir . '/lib/private/Share/Helper.php', 'OC\\Share\\SearchResultSorter' => $baseDir . '/lib/private/Share/SearchResultSorter.php', 'OC\\Share\\Share' => $baseDir . '/lib/private/Share/Share.php', + 'OC\\StreamHashFilter' => $baseDir . '/lib/private/StreamHashFilter.php', 'OC\\Streamer' => $baseDir . '/lib/private/Streamer.php', 'OC\\SubAdmin' => $baseDir . '/lib/private/SubAdmin.php', 'OC\\Support\\CrashReport\\Registry' => $baseDir . '/lib/private/Support/CrashReport/Registry.php', diff --git a/lib/composer/composer/autoload_static.php b/lib/composer/composer/autoload_static.php index d476460c82715..e8bcdbe47078b 100644 --- a/lib/composer/composer/autoload_static.php +++ b/lib/composer/composer/autoload_static.php @@ -1104,6 +1104,7 @@ class ComposerStaticInit53792487c5a8370acc0b06b1a864ff4c 'OC\\Share\\Helper' => __DIR__ . '/../../..' . '/lib/private/Share/Helper.php', 'OC\\Share\\SearchResultSorter' => __DIR__ . '/../../..' . '/lib/private/Share/SearchResultSorter.php', 'OC\\Share\\Share' => __DIR__ . '/../../..' . '/lib/private/Share/Share.php', + 'OC\\StreamHashFilter' => __DIR__ . '/../../..' . '/lib/private/StreamHashFilter.php', 'OC\\Streamer' => __DIR__ . '/../../..' . '/lib/private/Streamer.php', 'OC\\SubAdmin' => __DIR__ . '/../../..' . '/lib/private/SubAdmin.php', 'OC\\Support\\CrashReport\\Registry' => __DIR__ . '/../../..' . '/lib/private/Support/CrashReport/Registry.php',