diff --git a/Dockerfile b/Dockerfile index 0f6f659b4..b87a3e8ec 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,6 +30,9 @@ ENV HASHTOPOLIS_IMPORT_PATH=${HASHTOPOLIS_PATH}/import ENV HASHTOPOLIS_LOG_PATH=${HASHTOPOLIS_PATH}/log ENV HASHTOPOLIS_CONFIG_PATH=${HASHTOPOLIS_PATH}/config ENV HASHTOPOLIS_BINARIES_PATH=${HASHTOPOLIS_PATH}/binaries +ENV HASHTOPOLIS_TUS_PATH=/var/tmp/tus +ENV HASHTOPOLIS_TEMP_UPLOADS_PATH=${HASHTOPOLIS_TUS_PATH}/uploads +ENV HASHTOPOLIS_TEMP_META_PATH=${HASHTOPOLIS_TUS_PATH}/meta # Add support for TLS inspection corporate setups, see .env.sample for details ENV NODE_EXTRA_CA_CERTS=/etc/ssl/certs/ca-certificates.crt @@ -67,26 +70,24 @@ RUN echo "ServerTokens Prod" >> /etc/apache2/apache2.conf \ && echo "ServerSignature Off" >> /etc/apache2/apache2.conf -RUN mkdir -p ${HASHTOPOLIS_DOCUMENT_ROOT} \ - && mkdir ${HASHTOPOLIS_DOCUMENT_ROOT}/../../.git/ \ - && mkdir -p ${HASHTOPOLIS_PATH} \ - && chown www-data:www-data ${HASHTOPOLIS_PATH} \ - && chmod g+w ${HASHTOPOLIS_PATH} \ - && mkdir -p ${HASHTOPOLIS_FILES_PATH} \ - && chown www-data:www-data ${HASHTOPOLIS_FILES_PATH} \ - && chmod g+w ${HASHTOPOLIS_FILES_PATH} \ - && mkdir -p ${HASHTOPOLIS_IMPORT_PATH} \ - && chown www-data:www-data ${HASHTOPOLIS_IMPORT_PATH} \ - && chmod g+w ${HASHTOPOLIS_IMPORT_PATH} \ - && mkdir -p ${HASHTOPOLIS_LOG_PATH} \ - && chown www-data:www-data ${HASHTOPOLIS_LOG_PATH} \ - && chmod g+w ${HASHTOPOLIS_LOG_PATH} \ - && mkdir -p ${HASHTOPOLIS_CONFIG_PATH} \ - && chown www-data:www-data ${HASHTOPOLIS_CONFIG_PATH} \ - && chmod g+w ${HASHTOPOLIS_CONFIG_PATH} \ - && mkdir -p ${HASHTOPOLIS_BINARIES_PATH} \ - && chown www-data:www-data ${HASHTOPOLIS_BINARIES_PATH} \ - && chmod g+w ${HASHTOPOLIS_BINARIES_PATH} +RUN mkdir -p \ + ${HASHTOPOLIS_DOCUMENT_ROOT} \ + ${HASHTOPOLIS_DOCUMENT_ROOT}/../../.git/ \ + ${HASHTOPOLIS_PATH} \ + ${HASHTOPOLIS_FILES_PATH} \ + ${HASHTOPOLIS_IMPORT_PATH} \ + ${HASHTOPOLIS_LOG_PATH} \ + ${HASHTOPOLIS_CONFIG_PATH} \ + ${HASHTOPOLIS_BINARIES_PATH} \ + ${HASHTOPOLIS_TUS_PATH} \ + ${HASHTOPOLIS_TEMP_UPLOADS_PATH} \ + ${HASHTOPOLIS_TEMP_META_PATH} \ + && chown -R www-data:www-data \ + ${HASHTOPOLIS_PATH} \ + ${HASHTOPOLIS_TUS_PATH} \ + && chmod -R g+w \ + ${HASHTOPOLIS_PATH} \ + ${HASHTOPOLIS_TUS_PATH} COPY --from=prebuild /usr/local/cargo/bin/sqlx /usr/bin/ diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 72d5b2196..367142e97 100755 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -49,22 +49,23 @@ while :; do done echo "Database ready!" +directories=( + "${HASHTOPOLIS_FILES_PATH}" + "${HASHTOPOLIS_CONFIG_PATH}" + "${HASHTOPOLIS_LOG_PATH}" + "${HASHTOPOLIS_IMPORT_PATH}" + "${HASHTOPOLIS_BINARIES_PATH}" + "${HASHTOPOLIS_TUS_PATH}" + "${HASHTOPOLIS_TEMP_UPLOADS_PATH}" + "${HASHTOPOLIS_TEMP_META_PATH}" +) + echo "Setting up folders" -if [ ! -d ${HASHTOPOLIS_FILES_PATH} ];then - mkdir -p ${HASHTOPOLIS_FILES_PATH} && chown www-data:www-data ${HASHTOPOLIS_FILES_PATH} -fi -if [ ! -d ${HASHTOPOLIS_CONFIG_PATH} ];then - mkdir -p ${HASHTOPOLIS_CONFIG_PATH} && chown www-data:www-data ${HASHTOPOLIS_CONFIG_PATH} -fi -if [ ! -d ${HASHTOPOLIS_LOG_PATH} ];then - mkdir -p ${HASHTOPOLIS_LOG_PATH} && chown www-data:www-data ${HASHTOPOLIS_LOG_PATH} -fi -if [ ! -d ${HASHTOPOLIS_IMPORT_PATH} ];then - mkdir -p ${HASHTOPOLIS_IMPORT_PATH} && chown www-data:www-data ${HASHTOPOLIS_IMPORT_PATH} -fi -if [ ! -d ${HASHTOPOLIS_BINARIES_PATH} ];then - mkdir -p ${HASHTOPOLIS_BINARIES_PATH} && chown www-data:www-data ${HASHTOPOLIS_BINARIES_PATH} -fi +for dir in "${directories[@]}"; do + if [ ! -d "$dir" ];then + mkdir -p "$dir" && chown www-data:www-data "$dir" + fi +done # required to trigger the initialization echo "Start initialization process..." diff --git a/src/inc/Util.class.php b/src/inc/Util.class.php index a8a31ece9..273eb7d85 100755 --- a/src/inc/Util.class.php +++ b/src/inc/Util.class.php @@ -621,6 +621,21 @@ public static function checkTaskWrapperCompleted($taskWrapper) { } return true; } + + public static function cleaning() { + $entry = Factory::getStoredValueFactory()->get(DCleaning::LAST_CLEANING); + if ($entry == null) { + $entry = new StoredValue(DCleaning::LAST_CLEANING, 0); + Factory::getStoredValueFactory()->save($entry); + } + $time = time(); + if ($time - $entry->getVal() > 600) { + self::agentStatCleaning(); + self::zapCleaning(); + self::tusFileCleaning(); + Factory::getStoredValueFactory()->set($entry, StoredValue::VAL, $time); + } + } /** * Checks if it is longer than 10 mins since the last time it was checked if there are @@ -628,48 +643,69 @@ public static function checkTaskWrapperCompleted($taskWrapper) { * and old entries are deleted. */ public static function agentStatCleaning() { - $entry = Factory::getStoredValueFactory()->get(DStats::LAST_STAT_CLEANING); - if ($entry == null) { - $entry = new StoredValue(DStats::LAST_STAT_CLEANING, 0); - Factory::getStoredValueFactory()->save($entry); - } - if (time() - $entry->getVal() > 600) { - $lifetime = intval(SConfig::getInstance()->getVal(DConfig::AGENT_DATA_LIFETIME)); - if ($lifetime <= 0) { - $lifetime = 3600; - } - $qF = new QueryFilter(AgentStat::TIME, time() - $lifetime, "<="); - Factory::getAgentStatFactory()->massDeletion([Factory::FILTER => $qF]); - - $qF = new QueryFilter(Speed::TIME, time() - $lifetime, "<="); - Factory::getSpeedFactory()->massDeletion([Factory::FILTER => $qF]); - - Factory::getStoredValueFactory()->set($entry, StoredValue::VAL, time()); + $lifetime = intval(SConfig::getInstance()->getVal(DConfig::AGENT_DATA_LIFETIME)); + if ($lifetime <= 0) { + $lifetime = 3600; } + $qF = new QueryFilter(AgentStat::TIME, time() - $lifetime, "<="); + Factory::getAgentStatFactory()->massDeletion([Factory::FILTER => $qF]); + + $qF = new QueryFilter(Speed::TIME, time() - $lifetime, "<="); + Factory::getSpeedFactory()->massDeletion([Factory::FILTER => $qF]); + } /** * Used by the solver. Cleans the zap-queue */ public static function zapCleaning() { - $entry = Factory::getStoredValueFactory()->get(DZaps::LAST_ZAP_CLEANING); - if ($entry == null) { - $entry = new StoredValue(DZaps::LAST_ZAP_CLEANING, 0); - Factory::getStoredValueFactory()->save($entry); - } - if (time() - $entry->getVal() > 600) { - $zapFilter = new QueryFilter(Zap::SOLVE_TIME, time() - 600, "<="); - - // delete dependencies on AgentZap - $zaps = Factory::getZapFactory()->filter([Factory::FILTER => $zapFilter]); - $zapIds = Util::arrayOfIds($zaps); - $uS = new UpdateSet(AgentZap::LAST_ZAP_ID, null); - $qF = new ContainFilter(AgentZap::LAST_ZAP_ID, $zapIds); - Factory::getAgentZapFactory()->massUpdate([Factory::FILTER => $qF, Factory::UPDATE => $uS]); - - Factory::getZapFactory()->massDeletion([Factory::FILTER => $zapFilter]); - - Factory::getStoredValueFactory()->set($entry, StoredValue::VAL, time()); + $zapFilter = new QueryFilter(Zap::SOLVE_TIME, time() - 600, "<="); + + // delete dependencies on AgentZap + $zaps = Factory::getZapFactory()->filter([Factory::FILTER => $zapFilter]); + $zapIds = Util::arrayOfIds($zaps); + $uS = new UpdateSet(AgentZap::LAST_ZAP_ID, null); + $qF = new ContainFilter(AgentZap::LAST_ZAP_ID, $zapIds); + Factory::getAgentZapFactory()->massUpdate([Factory::FILTER => $qF, Factory::UPDATE => $uS]); + + Factory::getZapFactory()->massDeletion([Factory::FILTER => $zapFilter]); + } + + /** + * Cleans up stale TUS upload files. + * + * This method scans the TUS metadata directory for .meta files, reads their + * metadata to determine upload expiration, and removes expired metadata files + * together with their corresponding upload (.part) files. It performs file + * system operations and may delete files on disk. + */ + public static function tusFileCleaning() { + $tusDirectory = Factory::getStoredValueFactory()->get(DDirectories::TUS)->getVal(); + $uploadDirectory = $tusDirectory . DIRECTORY_SEPARATOR . "uploads" . DIRECTORY_SEPARATOR; + $metaDirectory = $tusDirectory . DIRECTORY_SEPARATOR . "meta" . DIRECTORY_SEPARATOR; + $expiration_time = time() + 3600; + if (file_exists($metaDirectory) && is_dir($metaDirectory)) { + if ($metaDirectoryHandler = opendir($metaDirectory)){ + while ($file = readdir($metaDirectoryHandler)) { + if (str_ends_with($file, ".meta")) { + $metaFile = $metaDirectory . $file; + $metadata = (array)json_decode(file_get_contents($metaFile), true) ; + if (!isset($metadata['upload_expires'])) { + continue; + } + if ($metadata['upload_expires'] > $expiration_time) { + $uploadFile = $uploadDirectory . pathinfo($file, PATHINFO_FILENAME) . ".part"; + if (file_exists($metaFile)) { + unlink($metaFile); + } + if (file_exists($uploadFile)){ + unlink($uploadFile); + } + } + } + } + closedir($metaDirectoryHandler); + } } } diff --git a/src/inc/api/APISendProgress.class.php b/src/inc/api/APISendProgress.class.php index c069e4faf..c2d9a243e 100644 --- a/src/inc/api/APISendProgress.class.php +++ b/src/inc/api/APISendProgress.class.php @@ -537,8 +537,7 @@ public function execute($QUERY = array()) { DServerLog::log(DServerLog::TRACE, "Checked zaps and sending new ones to agent", [$this->agent, $zaps]); break; } - Util::zapCleaning(); - Util::agentStatCleaning(); + Util::cleaning(); $this->sendResponse(array( PResponseSendProgress::ACTION => PActions::SEND_PROGRESS, PResponseSendProgress::RESPONSE => PValues::SUCCESS, diff --git a/src/inc/apiv2/helper/importFile.routes.php b/src/inc/apiv2/helper/importFile.routes.php index 2e361db80..fec46de03 100644 --- a/src/inc/apiv2/helper/importFile.routes.php +++ b/src/inc/apiv2/helper/importFile.routes.php @@ -37,14 +37,20 @@ public function getRequiredPermissions(string $method): array { return []; } - static function getUploadPath(string $id): string { - return "/tmp/" . $id . '.part'; - } - - static function getMetaPath(string $id): string { - return "/tmp/" . $id . '.meta'; - } - +static function getUploadPath(string $id): string { + return Factory::getStoredValueFactory()->get(DDirectories::TUS)->getVal() . DIRECTORY_SEPARATOR . 'uploads' . + DIRECTORY_SEPARATOR . basename($id) . ".part"; +} + +static function getMetaPath(string $id): string { + return Factory::getStoredValueFactory()->get(DDirectories::TUS)->getVal() . DIRECTORY_SEPARATOR . 'meta' + . DIRECTORY_SEPARATOR . basename($id) . ".meta"; +} + +static function getImportPath(string $id): string { + return Factory::getStoredValueFactory()->get(DDirectories::IMPORT)->getVal() . DIRECTORY_SEPARATOR . basename($id); +} + /** * Import file has no POST parameters */ @@ -52,11 +58,6 @@ public function getFormFields(): array { return []; } - - static function getImportPath(string $id): string { - return Factory::getStoredValueFactory()->get(DDirectories::IMPORT)->getVal() . "/" . $id; - } - static function getChecksumAlgorithm(): array { return ['md5', 'sha1', 'crc32']; } @@ -86,8 +87,10 @@ function actionPost(array $data): object|array|null { * And to retrieve the upload status. */ function processHead(Request $request, Response $response, array $args): Response { - // TODO return 404 or 410 if entry is not found $filename = self::getUploadPath($args['id']); + if (!is_file($filename)) { + return $response->withStatus(404); + } $currentSize = filesize($filename); $ds = self::getMetaStorage($args['id']); @@ -155,6 +158,10 @@ function processPost(Request $request, Response $response, array $args): Respons $list = explode(",", $update["upload_metadata_raw"]); foreach ($list as $item) { list($key, $b64val) = explode(" ", $item); + if (!isset($b64val)) { + $response->getBody()->write("Error Upload-Metadata, should be a key value pair that is separated by a space, no value has been provided"); + return $response->withStatus(400); + } if (($val = base64_decode($b64val, true)) === false) { $response->getBody()->write("Error Upload-Metadata '$key' invalid base64 encoding"); return $response->withStatus(400); @@ -163,7 +170,7 @@ function processPost(Request $request, Response $response, array $args): Respons } } // TODO: Should filename be mandatory? - if (array_key_exists('filename', $update_metadata)) { + if (isset($update_metadata) && array_key_exists('filename', $update_metadata)) { $filename = $update_metadata['filename']; /* Generate unique upload identifier */ $id = date("YmdHis") . "-" . md5($filename); @@ -178,6 +185,10 @@ function processPost(Request $request, Response $response, array $args): Respons } $update["upload_metadata"] = $update_metadata; + if ($request->hasHeader('Upload-Defer-Length') && $request->hasHeader('Upload-Length')) { + $response->getBody()->write('Error: Cannot provide both Upload-Length and Upload-Defer-Length'); + return $response->withStatus(400); + } if ($request->hasHeader('Upload-Defer-Length')) { if ($request->getHeader('Upload-Defer-Length')[0] == "1") { $update["upload_defer_length"] = true; @@ -252,9 +263,12 @@ function processPatch(Request $request, Response $response, array $args): Respon /* Validate if upload time is still valid */ $now = new DateTimeImmutable(); + if (!isset($ds['upload_expires'])) { + throw new HttpError("The meta file of this upload is incorrect"); + } $dt = (new DateTime())->setTimeStamp($ds['upload_expires']); if (($dt->getTimestamp() - $now->getTimestamp()) <= 0) { - // TODO: Remove expired uploads + Util::tusFileCleaning(); $response->getBody()->write('Upload token expired'); return $response->withStatus(410); } @@ -302,9 +316,12 @@ function processPatch(Request $request, Response $response, array $args): Respon self::updateStorage($args['id'], $update); } } - - file_put_contents($filename, $chunk, FILE_APPEND); - + + if (file_put_contents($filename, $chunk, FILE_APPEND) === false) { + $response->getBody()->write('Failed to write to file'); + return $response->withStatus(400); + } + clearstatcache(); $newSize = filesize($filename); @@ -333,7 +350,8 @@ function processPatch(Request $request, Response $response, array $args): Respon else { $statusMsg = "Next chunk please"; } - + + $dt = (new DateTime())->setTimeStamp($ds['upload_expires']); $response->getBody()->write($statusMsg); return $response->withStatus(204) ->withHeader("Tus-Resumable", "1.0.0") @@ -342,18 +360,32 @@ function processPatch(Request $request, Response $response, array $args): Respon ->withHeader('Upload-Expires', $dt->format(DateTimeInterface::RFC7231)) ->WithHeader("Access-Control-Expose-Headers", "Tus-Resumable, Upload-Length, Upload-Offset"); } - - /** - * Endpoint to delete the file - */ + function processDelete(Request $request, Response $response, array $args): Response { - // // TODO delete file - - // // TODO return 404 or 410 if entry is not found - return $response->withStatus(204) - ->withHeader("Tus-Resumable", "1.0.0") - ->WithHeader("Access-Control-Expose-Headers", "Tus-Resumable"); + /* Return 404 if entry is not found */ + $filename_upload = self::getUploadPath($args['id']); + $filename_meta = self::getMetaPath($args['id']); + $uploadExists = file_exists($filename_upload); + $metaExists = file_exists($filename_meta); + if (!$uploadExists && !$metaExists) { + throw new HttpError("Upload ID doesnt exists"); + } + if ($uploadExists) { + $isDeletedUpload = unlink($filename_upload); + } + if ($metaExists) { + $isDeletedMeta = unlink($filename_meta); + } + + if (!$isDeletedMeta || !$isDeletedUpload) { + throw new HttpError("Something went wrong while deleting the files"); + } + + return $response->withStatus(204) + ->withHeader("Tus-Resumable", "1.0.0") + ->WithHeader("Access-Control-Expose-Headers", "Tus-Resumable"); } + static public function register($app): void { $me = get_called_class(); @@ -373,7 +405,7 @@ static public function register($app): void { $group->post('', $me . ":processPost")->setName($me . ":processPost"); }); - + $app->group($baseUri . "/{id:[0-9]{14}-[0-9a-f]{32}}", function (RouteCollectorProxy $group) use ($me) { /* Allow preflight requests */ $group->options('', function (Request $request, Response $response, array $args): Response { diff --git a/src/inc/confv2.php b/src/inc/confv2.php index d13484e72..fe94080c1 100644 --- a/src/inc/confv2.php +++ b/src/inc/confv2.php @@ -10,7 +10,8 @@ "files" => dirname(__FILE__) . "/../files/", "import" => dirname(__FILE__) . "/../import/", "log" => dirname(__FILE__) . "/../log/", - "config" => dirname(__FILE__) . "/../config/" + "config" => dirname(__FILE__) . "/../config/", + "tus" => "/var/tmp/tus/", ]; } @@ -48,7 +49,8 @@ "files" => "/usr/local/share/hashtopolis/files", "import" => "/usr/local/share/hashtopolis/import", "log" => "/usr/local/share/hashtopolis/log", - "config" => "/usr/local/share/hashtopolis/config" + "config" => "/usr/local/share/hashtopolis/config", + "tus" => "/var/tmp/tus/", ]; // update from env if set @@ -61,6 +63,9 @@ if (getenv('HASHTOPOLIS_LOG_PATH') !== false) { $DIRECTORIES["log"] = getenv('HASHTOPOLIS_LOG_PATH'); } + if (getenv('HASHTOPOLIS_TUS_PATH') !== false) { + $DIRECTORIES["tus"] = getenv('HASHTOPOLIS_TUS_PATH'); + } } // load data // test if config file exists diff --git a/src/inc/defines/global.php b/src/inc/defines/global.php index 9323d88ac..662aa3bb8 100644 --- a/src/inc/defines/global.php +++ b/src/inc/defines/global.php @@ -4,8 +4,8 @@ class DLimits { const ACCESS_GROUP_MAX_LENGTH = 50; } -class DZaps { - const LAST_ZAP_CLEANING = "lastZapCleaning"; +class DCleaning { + const LAST_CLEANING = "lastCleaning"; } class DDirectories { @@ -13,6 +13,7 @@ class DDirectories { const IMPORT = "directory_import"; const LOG = "directory_log"; const CONFIG = "directory_config"; + const TUS = "directory_tus"; } // log entry types @@ -36,8 +37,6 @@ class DStats { const TASKS_FINISHED = "tasksFinished"; const TASKS_RUNNING = "tasksRunning"; const TASKS_QUEUED = "tasksQueued"; - - const LAST_STAT_CLEANING = "lastStatCleaning"; } class DPrince { diff --git a/src/inc/startup/setup.php b/src/inc/startup/setup.php index b55084343..233cb8558 100755 --- a/src/inc/startup/setup.php +++ b/src/inc/startup/setup.php @@ -114,3 +114,4 @@ Util::checkDataDirectory(DDirectories::IMPORT, $DIRECTORIES['import']); Util::checkDataDirectory(DDirectories::LOG, $DIRECTORIES['log']); Util::checkDataDirectory(DDirectories::CONFIG, $DIRECTORIES['config']); +Util::checkDataDirectory(DDirectories::TUS, $DIRECTORIES['tus']); \ No newline at end of file