From 4f96952ce73affa8bcf46668a5e643a86b1fde6f Mon Sep 17 00:00:00 2001 From: Andrey Borysenko Date: Wed, 27 Dec 2023 18:17:43 +0200 Subject: [PATCH 01/11] WIP: STT implementation with custom ProviderID Signed-off-by: Andrey Borysenko --- appinfo/routes.php | 4 + docs/tech_details/api/speechtotext.rst | 42 +++++ lib/AppInfo/Application.php | 10 ++ lib/Controller/SpeechToTextController.php | 76 +++++++++ .../ExAppSpeechToTextProvider.php | 63 ++++++++ .../ExAppSpeechToTextProviderMapper.php | 66 ++++++++ lib/Migration/Version1005Date202312271744.php | 56 +++++++ lib/Service/SpeechToTextService.php | 146 ++++++++++++++++++ 8 files changed, 463 insertions(+) create mode 100644 docs/tech_details/api/speechtotext.rst create mode 100644 lib/Controller/SpeechToTextController.php create mode 100644 lib/Db/SpeechToText/ExAppSpeechToTextProvider.php create mode 100644 lib/Db/SpeechToText/ExAppSpeechToTextProviderMapper.php create mode 100644 lib/Migration/Version1005Date202312271744.php create mode 100644 lib/Service/SpeechToTextService.php diff --git a/appinfo/routes.php b/appinfo/routes.php index e2d72d21..436cd7c5 100644 --- a/appinfo/routes.php +++ b/appinfo/routes.php @@ -100,5 +100,9 @@ ['name' => 'OCSUi#setExAppStyle', 'url' => '/api/v1/ui/style', 'verb' => 'POST'], ['name' => 'OCSUi#deleteExAppStyle', 'url' => '/api/v1/ui/style', 'verb' => 'DELETE'], ['name' => 'OCSUi#getExAppStyle', 'url' => '/api/v1/ui/style', 'verb' => 'GET'], + + // Speech-To-Text + ['name' => 'speechToText#registerProvider', 'url' => '/api/v1/speech_to_text', 'verb' => 'POST'], + ['name' => 'speechToText#unregisterProvider', 'url' => '/api/v1/speech_to_text', 'verb' => 'DELETE'], ], ]; diff --git a/docs/tech_details/api/speechtotext.rst b/docs/tech_details/api/speechtotext.rst new file mode 100644 index 00000000..66c3b1dc --- /dev/null +++ b/docs/tech_details/api/speechtotext.rst @@ -0,0 +1,42 @@ +============== +Speech-To-Text +============== + +AppAPI provides a Speech-To-Text (STT) service +that can be used to register ExApp as a custom STT model and transcribe audio files via it. + +Registering ExApp STT provider (OCS) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +OCS endpoint: ``POST /apps/app_api/api/v1/speech_to_text`` + +Request data +************ + +.. code-block:: json + { + "name": "unique_provider_name", + "display_name": "Provider Display Name", + "action_handler_route": "/handler_route_on_ex_app", + } +Response +******** + +On successful registration response with status code 200 is returned. + +Unregistering ExApp STT provider (OCS) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +OCS endpoint: ``DELETE /apps/app_api/api/v1/speech_to_text`` + +Request data +************ + +.. code-block:: json + { + "name": "unique_provider_name", + } +Response +******** + +On successful unregister response with status code 200 is returned. diff --git a/lib/AppInfo/Application.php b/lib/AppInfo/Application.php index 5d12ce06..9265db8d 100644 --- a/lib/AppInfo/Application.php +++ b/lib/AppInfo/Application.php @@ -15,6 +15,7 @@ use OCA\AppAPI\Notifications\ExAppNotifier; use OCA\AppAPI\Profiler\AppAPIDataCollector; use OCA\AppAPI\PublicCapabilities; +use OCA\AppAPI\Service\SpeechToTextService; use OCA\AppAPI\Service\UI\TopMenuService; use OCA\DAV\Events\SabrePluginAuthInitEvent; use OCA\Files\Event\LoadAdditionalScriptsEvent; @@ -58,6 +59,15 @@ public function register(IRegistrationContext $context): void { $context->registerEventListener(UserDeletedEvent::class, UserDeletedListener::class); $context->registerNotifierService(ExAppNotifier::class); $context->registerNotifierService(ExAppAdminNotifier::class); + + // Dynamic anonymous providers registration + $container = $this->getContainer(); + try { + /** @var SpeechToTextService $speechToTextService */ + $speechToTextService = $container->get(SpeechToTextService::class); + $speechToTextService->registerExAppSpeechToTextProviders($context); + } catch (NotFoundExceptionInterface|ContainerExceptionInterface) { + } } public function boot(IBootContext $context): void { diff --git a/lib/Controller/SpeechToTextController.php b/lib/Controller/SpeechToTextController.php new file mode 100644 index 00000000..6e150793 --- /dev/null +++ b/lib/Controller/SpeechToTextController.php @@ -0,0 +1,76 @@ +request = $request; + } + + /** + * @param string $name + * @param string $displayName + * @param string $actionHandlerRoute + * + * @throws OCSBadRequestException + * @return Response + */ + #[NoCSRFRequired] + #[PublicPage] + #[AppAPIAuth] + public function registerProvider(string $name, string $displayName, string $actionHandlerRoute): Response { + $appId = $this->request->getHeader('EX-APP-ID'); + $exApp = $this->service->getExApp($appId); + + $provider = $this->speechToTextService->registerSpeechToTextProvider($exApp, $name, $displayName, $actionHandlerRoute); + + if ($provider === null) { + throw new OCSBadRequestException('Failed to register STT provider'); + } + + return new DataResponse(); + } + + /** + * @param string $name + * + * @throws OCSBadRequestException + * @return Response + */ + #[NoCSRFRequired] + #[PublicPage] + #[AppAPIAuth] + public function unregisterProvider(string $name): Response { + $appId = $this->request->getHeader('EX-APP-ID'); + $exApp = $this->service->getExApp($appId); + $unregistered = $this->speechToTextService->unregisterSpeechToTextProvider($exApp, $name); + + if ($unregistered === null) { + throw new OCSBadRequestException('Failed to unregister STT provider'); + } + + return new DataResponse(); + } +} diff --git a/lib/Db/SpeechToText/ExAppSpeechToTextProvider.php b/lib/Db/SpeechToText/ExAppSpeechToTextProvider.php new file mode 100644 index 00000000..bb19c851 --- /dev/null +++ b/lib/Db/SpeechToText/ExAppSpeechToTextProvider.php @@ -0,0 +1,63 @@ +addType('appid', 'string'); + $this->addType('name', 'string'); + $this->addType('displayName', 'string'); + $this->addType('actionHandlerRoute', 'string'); + + if (isset($params['id'])) { + $this->setId($params['id']); + } + if (isset($params['appid'])) { + $this->setAppid($params['appid']); + } + if (isset($params['name'])) { + $this->setName($params['name']); + } + if (isset($params['display_name'])) { + $this->setDisplayName($params['display_name']); + } + if (isset($params['action_handler_route'])) { + $this->setActionHandlerRoute($params['action_handler_route']); + } + } + + public function jsonSerialize(): array { + return [ + 'id' => $this->getId(), + 'appid' => $this->getAppid(), + 'name' => $this->getName(), + 'display_name' => $this->getDisplayName(), + 'action_handler_route' => $this->getActionHandlerRoute(), + ]; + } +} diff --git a/lib/Db/SpeechToText/ExAppSpeechToTextProviderMapper.php b/lib/Db/SpeechToText/ExAppSpeechToTextProviderMapper.php new file mode 100644 index 00000000..01802eeb --- /dev/null +++ b/lib/Db/SpeechToText/ExAppSpeechToTextProviderMapper.php @@ -0,0 +1,66 @@ + + */ +class ExAppSpeechToTextProviderMapper extends QBMapper { + public function __construct(IDBConnection $db) { + parent::__construct($db, 'ex_apps_speech_to_text'); + } + + /** + * @throws Exception + */ + public function findAll(int $limit = null, int $offset = null): array { + $qb = $this->db->getQueryBuilder(); + $qb->select('*') + ->from($this->tableName) + ->setMaxResults($limit) + ->setFirstResult($offset); + return $this->findEntities($qb); + } + + /** + * @param string $appId + * + * @throws Exception + * @return ExAppSpeechToTextProvider[] + */ + public function findByAppid(string $appId): array { + $qb = $this->db->getQueryBuilder(); + return $this->findEntities($qb->select('*') + ->from($this->tableName) + ->where($qb->expr()->eq('appid', $qb->createNamedParameter($appId), IQueryBuilder::PARAM_STR)) + ); + } + + /** + * @param string $appId + * @param string $name + * + * @throws DoesNotExistException + * @throws Exception + * @throws MultipleObjectsReturnedException + * + * @return ExAppSpeechToTextProvider + */ + public function findByAppidName(string $appId, string $name): ExAppSpeechToTextProvider { + $qb = $this->db->getQueryBuilder(); + return $this->findEntity($qb->select('*') + ->from($this->tableName) + ->where($qb->expr()->eq('appid', $qb->createNamedParameter($appId), IQueryBuilder::PARAM_STR)) + ->andWhere($qb->expr()->eq('name', $qb->createNamedParameter($name), IQueryBuilder::PARAM_STR)) + ); + } +} diff --git a/lib/Migration/Version1005Date202312271744.php b/lib/Migration/Version1005Date202312271744.php new file mode 100644 index 00000000..4a43a8a8 --- /dev/null +++ b/lib/Migration/Version1005Date202312271744.php @@ -0,0 +1,56 @@ +hasTable('ex_apps_speech_to_text')) { + $table = $schema->createTable('ex_apps_speech_to_text'); + + $table->addColumn('id', Types::BIGINT, [ + 'autoincrement' => true, + 'notnull' => true, + ]); + $table->addColumn('appid', Types::STRING, [ + 'notnull' => true, + 'length' => 32, + ]); + $table->addColumn('name', Types::STRING, [ + 'notnull' => true, + 'length' => 64, + ]); + $table->addColumn('display_name', Types::STRING, [ + 'notnull' => true, + 'length' => 64, + ]); + // ExApp route to forward the action + $table->addColumn('action_handler_route', Types::STRING, [ + 'notnull' => true, + 'length' => 64, + ]); + + $table->setPrimaryKey(['id'], 'ex_apps_speech_to_text_id'); + $table->addUniqueIndex(['appid', 'name'], 'speech_to_text_appid_name'); + } + + return $schema; + } +} diff --git a/lib/Service/SpeechToTextService.php b/lib/Service/SpeechToTextService.php new file mode 100644 index 00000000..9a7a8f8e --- /dev/null +++ b/lib/Service/SpeechToTextService.php @@ -0,0 +1,146 @@ +cache = $cacheFactory->createDistributed(Application::APP_ID . '/ex_apps_speech_to_text'); + } + + public function getSpeechToTextProviders(): array { + $cacheKey = '/ex_app_speech_to_text_providers'; + $cached = $this->cache->get($cacheKey); + if ($cached !== null) { + return array_map(function ($cachedEntry) { + return $cachedEntry instanceof ExAppSpeechToTextProvider ? $cachedEntry : new ExAppSpeechToTextProvider($cachedEntry); + }, $cached); + } + + $providers = $this->speechToTextProviderMapper->findAll(); + $this->cache->set($cacheKey, $providers); + return $providers; + } + + public function getExAppSpeechToTextProvider(ExApp $exApp, string $name): ?ExAppSpeechToTextProvider { + $cacheKey = '/ex_app_speech_to_text_providers/' . $exApp->getAppid() . '/' . $name; + $cached = $this->cache->get($cacheKey); + if ($cached !== null) { + return $cached instanceof ExAppSpeechToTextProvider ? $cached : new ExAppSpeechToTextProvider($cached); + } + + $provider = $this->speechToTextProviderMapper->findByAppidName($exApp->getAppid(), $name); + $this->cache->set($cacheKey, $provider); + return $provider; + } + + public function registerSpeechToTextProvider(ExApp $exApp, string $name, string $displayName, string $actionHandlerRoute): ?ExAppSpeechToTextProvider { + $provider = new ExAppSpeechToTextProvider([ + 'appid' => $exApp->getAppid(), + 'name' => $name, + 'display_name' => $displayName, + 'action_handler_route' => $actionHandlerRoute, + ]); + try { + $this->speechToTextProviderMapper->insert($provider); + $this->cache->remove('/ex_app_speech_to_text_providers'); + return $provider; + } catch (Exception $e) { + $this->logger->error('Failed to register SpeechToText provider', ['exception' => $e]); + return null; + } + } + + public function unregisterSpeechToTextProvider(ExApp $exApp, string $name): ?ExAppSpeechToTextProvider { + $provider = $this->getExAppSpeechToTextProvider($exApp, $name); + if ($provider === null) { + return null; + } + try { + $this->speechToTextProviderMapper->delete($provider); + $this->cache->remove('/ex_app_speech_to_text_providers'); + return $provider; + } catch (Exception $e) { + $this->logger->error('Failed to unregister STT provider', ['exception' => $e]); + return null; + } + } + + /** + * Register ExApp anonymous providers implementations of ISpeechToTextProviderWithId + * so that they can be used as regular providers in DI container + * + * @param IRegistrationContext $context + * + * @return void + */ + public function registerExAppSpeechToTextProviders(IRegistrationContext &$context): void { + $exAppsProviders = $this->getSpeechToTextProviders(); + /** @var ExAppSpeechToTextProvider $exAppProvider */ + foreach ($exAppsProviders as $exAppProvider) { + $class = '\\OCA\\AppAPI\\' . $exAppProvider->getAppid() . '_' . $exAppProvider->getName(); + $sttProvider = $this->getAnonymousExAppProvider($exAppProvider, $class); + $context->registerService($class, function () use ($sttProvider) { + return $sttProvider; + }); + $context->registerSpeechToTextProvider($class); + } + } + + private function getAnonymousExAppProvider(ExAppSpeechToTextProvider $provider, string $class): ?ISpeechToTextProviderWithId { + return new class ($this->service, $provider, $this->userId, $class) implements ISpeechToTextProviderWithId { + public function __construct( + private AppAPIService $service, + private ExAppSpeechToTextProvider $sttProvider, + private readonly ?string $userId, + private readonly string $class, + ) { + } + + public function getId(): string { + return $this->class; + } + + public function getName(): string { + return $this->sttProvider->getDisplayName(); + } + + public function transcribeFile(File $file): string { + $route = $this->sttProvider->getActionHandlerRoute(); + $exApp = $this->service->getExApp($this->sttProvider->getAppid()); + + $response = $this->service->requestToExApp($exApp, $route, $this->userId, 'POST', [ + 'fileid' => $file->getId(), + ]); + + if ($response->getStatusCode() !== Http::STATUS_OK) { + throw new \Exception('Failed to transcribe file'); + } + + return $response->getBody(); + } + }; + } +} From 65c9634829ba5869d5bc3006419783df4cee9a41 Mon Sep 17 00:00:00 2001 From: Andrey Borysenko Date: Wed, 27 Dec 2023 18:31:43 +0200 Subject: [PATCH 02/11] fix psalm, php-cs, docs Signed-off-by: Andrey Borysenko --- docs/tech_details/api/index.rst | 1 + docs/tech_details/api/speechtotext.rst | 6 ++++++ lib/Service/SpeechToTextService.php | 5 ++++- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/docs/tech_details/api/index.rst b/docs/tech_details/api/index.rst index dc7999f5..8374f026 100644 --- a/docs/tech_details/api/index.rst +++ b/docs/tech_details/api/index.rst @@ -18,4 +18,5 @@ AppAPI Nextcloud APIs topmenu notifications talkbots + speechtotext other_ocs diff --git a/docs/tech_details/api/speechtotext.rst b/docs/tech_details/api/speechtotext.rst index 66c3b1dc..9034a46a 100644 --- a/docs/tech_details/api/speechtotext.rst +++ b/docs/tech_details/api/speechtotext.rst @@ -14,11 +14,14 @@ Request data ************ .. code-block:: json + { "name": "unique_provider_name", "display_name": "Provider Display Name", "action_handler_route": "/handler_route_on_ex_app", } + + Response ******** @@ -33,9 +36,12 @@ Request data ************ .. code-block:: json + { "name": "unique_provider_name", } + + Response ******** diff --git a/lib/Service/SpeechToTextService.php b/lib/Service/SpeechToTextService.php index 9a7a8f8e..5ef188a9 100644 --- a/lib/Service/SpeechToTextService.php +++ b/lib/Service/SpeechToTextService.php @@ -109,8 +109,11 @@ public function registerExAppSpeechToTextProviders(IRegistrationContext &$contex } } + /** + * @psalm-suppress UndefinedClass, MissingDependency, InvalidReturnStatement, InvalidReturnType + */ private function getAnonymousExAppProvider(ExAppSpeechToTextProvider $provider, string $class): ?ISpeechToTextProviderWithId { - return new class ($this->service, $provider, $this->userId, $class) implements ISpeechToTextProviderWithId { + return new class($this->service, $provider, $this->userId, $class) implements ISpeechToTextProviderWithId { public function __construct( private AppAPIService $service, private ExAppSpeechToTextProvider $sttProvider, From d4d6f6fad84323a6fc781dbbff83f88adf28c0e5 Mon Sep 17 00:00:00 2001 From: Andrey Borysenko Date: Thu, 28 Dec 2023 15:12:42 +0200 Subject: [PATCH 03/11] fix: broken AppAPI auth Signed-off-by: Andrey Borysenko --- lib/AppInfo/Application.php | 3 ++- lib/Service/SpeechToTextService.php | 20 +++++++++++--------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/lib/AppInfo/Application.php b/lib/AppInfo/Application.php index 9265db8d..6b270fae 100644 --- a/lib/AppInfo/Application.php +++ b/lib/AppInfo/Application.php @@ -4,6 +4,7 @@ namespace OCA\AppAPI\AppInfo; +use OC\Server; use OCA\AppAPI\Capabilities; use OCA\AppAPI\DavPlugin; use OCA\AppAPI\Listener\LoadFilesPluginListener; @@ -65,7 +66,7 @@ public function register(IRegistrationContext $context): void { try { /** @var SpeechToTextService $speechToTextService */ $speechToTextService = $container->get(SpeechToTextService::class); - $speechToTextService->registerExAppSpeechToTextProviders($context); + $speechToTextService->registerExAppSpeechToTextProviders($context, $container->getServer()); } catch (NotFoundExceptionInterface|ContainerExceptionInterface) { } } diff --git a/lib/Service/SpeechToTextService.php b/lib/Service/SpeechToTextService.php index 5ef188a9..c5520e4e 100644 --- a/lib/Service/SpeechToTextService.php +++ b/lib/Service/SpeechToTextService.php @@ -14,6 +14,7 @@ use OCP\Files\File; use OCP\ICache; use OCP\ICacheFactory; +use OCP\IServerContainer; use OCP\SpeechToText\ISpeechToTextProviderWithId; use Psr\Log\LoggerInterface; @@ -22,7 +23,6 @@ class SpeechToTextService { public function __construct( ICacheFactory $cacheFactory, - private readonly AppAPIService $service, private readonly ExAppSpeechToTextProviderMapper $speechToTextProviderMapper, private readonly LoggerInterface $logger, private readonly ?string $userId, @@ -96,12 +96,12 @@ public function unregisterSpeechToTextProvider(ExApp $exApp, string $name): ?ExA * * @return void */ - public function registerExAppSpeechToTextProviders(IRegistrationContext &$context): void { + public function registerExAppSpeechToTextProviders(IRegistrationContext &$context, IServerContainer $serverContainer): void { $exAppsProviders = $this->getSpeechToTextProviders(); /** @var ExAppSpeechToTextProvider $exAppProvider */ foreach ($exAppsProviders as $exAppProvider) { - $class = '\\OCA\\AppAPI\\' . $exAppProvider->getAppid() . '_' . $exAppProvider->getName(); - $sttProvider = $this->getAnonymousExAppProvider($exAppProvider, $class); + $class = '\\OCA\\AppAPI\\' . $exAppProvider->getAppid() . '\\' . $exAppProvider->getName(); + $sttProvider = $this->getAnonymousExAppProvider($exAppProvider, $serverContainer, $class); $context->registerService($class, function () use ($sttProvider) { return $sttProvider; }); @@ -112,11 +112,12 @@ public function registerExAppSpeechToTextProviders(IRegistrationContext &$contex /** * @psalm-suppress UndefinedClass, MissingDependency, InvalidReturnStatement, InvalidReturnType */ - private function getAnonymousExAppProvider(ExAppSpeechToTextProvider $provider, string $class): ?ISpeechToTextProviderWithId { - return new class($this->service, $provider, $this->userId, $class) implements ISpeechToTextProviderWithId { + private function getAnonymousExAppProvider(ExAppSpeechToTextProvider $provider, IServerContainer $serverContainer, string $class): ?ISpeechToTextProviderWithId { + return new class($provider, $serverContainer, $this->userId, $class) implements ISpeechToTextProviderWithId { public function __construct( - private AppAPIService $service, private ExAppSpeechToTextProvider $sttProvider, + // We need this to delay the instantiation of AppAPIService during registration to avoid conflicts + private IServerContainer $serverContainer, // TODO: Extract needed methods from AppAPIService to be able to use it everytime private readonly ?string $userId, private readonly string $class, ) { @@ -132,9 +133,10 @@ public function getName(): string { public function transcribeFile(File $file): string { $route = $this->sttProvider->getActionHandlerRoute(); - $exApp = $this->service->getExApp($this->sttProvider->getAppid()); + $service = $this->serverContainer->get(AppAPIService::class); + $exApp = $service->getExApp($this->sttProvider->getAppid()); - $response = $this->service->requestToExApp($exApp, $route, $this->userId, 'POST', [ + $response = $service->requestToExApp($exApp, $route, $this->userId, 'POST', [ 'fileid' => $file->getId(), ]); From 92351705cc24161752e73f088fb454b9dddb7296 Mon Sep 17 00:00:00 2001 From: Andrey Borysenko Date: Thu, 28 Dec 2023 15:13:48 +0200 Subject: [PATCH 04/11] fix php-cs Signed-off-by: Andrey Borysenko --- lib/AppInfo/Application.php | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/AppInfo/Application.php b/lib/AppInfo/Application.php index 6b270fae..a97e0782 100644 --- a/lib/AppInfo/Application.php +++ b/lib/AppInfo/Application.php @@ -4,7 +4,6 @@ namespace OCA\AppAPI\AppInfo; -use OC\Server; use OCA\AppAPI\Capabilities; use OCA\AppAPI\DavPlugin; use OCA\AppAPI\Listener\LoadFilesPluginListener; From e2eca6eea002c9c72831531ce4dfba55b7421f83 Mon Sep 17 00:00:00 2001 From: Alexander Piskun Date: Fri, 29 Dec 2023 10:27:14 +0300 Subject: [PATCH 05/11] unified the code --- appinfo/routes.php | 4 ++-- docs/tech_details/api/speechtotext.rst | 9 ++++--- lib/Controller/SpeechToTextController.php | 24 +++---------------- .../ExAppSpeechToTextProvider.php | 16 ++++++------- lib/Migration/Version1005Date202312271744.php | 13 +++++----- 5 files changed, 22 insertions(+), 44 deletions(-) diff --git a/appinfo/routes.php b/appinfo/routes.php index 436cd7c5..3a4e90ca 100644 --- a/appinfo/routes.php +++ b/appinfo/routes.php @@ -102,7 +102,7 @@ ['name' => 'OCSUi#getExAppStyle', 'url' => '/api/v1/ui/style', 'verb' => 'GET'], // Speech-To-Text - ['name' => 'speechToText#registerProvider', 'url' => '/api/v1/speech_to_text', 'verb' => 'POST'], - ['name' => 'speechToText#unregisterProvider', 'url' => '/api/v1/speech_to_text', 'verb' => 'DELETE'], + ['name' => 'speechToText#registerProvider', 'url' => '/api/v1/provider/speech_to_text', 'verb' => 'POST'], + ['name' => 'speechToText#unregisterProvider', 'url' => '/api/v1/provider/speech_to_text', 'verb' => 'DELETE'], ], ]; diff --git a/docs/tech_details/api/speechtotext.rst b/docs/tech_details/api/speechtotext.rst index 9034a46a..49117fe8 100644 --- a/docs/tech_details/api/speechtotext.rst +++ b/docs/tech_details/api/speechtotext.rst @@ -2,13 +2,12 @@ Speech-To-Text ============== -AppAPI provides a Speech-To-Text (STT) service -that can be used to register ExApp as a custom STT model and transcribe audio files via it. +AppAPI provides a Speech-To-Text (STT) provider registration API for the ExApps. Registering ExApp STT provider (OCS) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -OCS endpoint: ``POST /apps/app_api/api/v1/speech_to_text`` +OCS endpoint: ``POST /apps/app_api/api/v1/provider/speech_to_text`` Request data ************ @@ -18,7 +17,7 @@ Request data { "name": "unique_provider_name", "display_name": "Provider Display Name", - "action_handler_route": "/handler_route_on_ex_app", + "action_handler": "/handler_route_on_ex_app", } @@ -30,7 +29,7 @@ On successful registration response with status code 200 is returned. Unregistering ExApp STT provider (OCS) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -OCS endpoint: ``DELETE /apps/app_api/api/v1/speech_to_text`` +OCS endpoint: ``DELETE /apps/app_api/api/v1/provider/speech_to_text`` Request data ************ diff --git a/lib/Controller/SpeechToTextController.php b/lib/Controller/SpeechToTextController.php index 6e150793..4b72704a 100644 --- a/lib/Controller/SpeechToTextController.php +++ b/lib/Controller/SpeechToTextController.php @@ -8,11 +8,11 @@ use OCA\AppAPI\Attribute\AppAPIAuth; use OCA\AppAPI\Service\AppAPIService; use OCA\AppAPI\Service\SpeechToTextService; +use OCP\AppFramework\Http; use OCP\AppFramework\Http\Attribute\NoCSRFRequired; use OCP\AppFramework\Http\Attribute\PublicPage; use OCP\AppFramework\Http\DataResponse; use OCP\AppFramework\Http\Response; -use OCP\AppFramework\OCS\OCSBadRequestException; use OCP\AppFramework\OCSController; use OCP\IRequest; @@ -29,14 +29,6 @@ public function __construct( $this->request = $request; } - /** - * @param string $name - * @param string $displayName - * @param string $actionHandlerRoute - * - * @throws OCSBadRequestException - * @return Response - */ #[NoCSRFRequired] #[PublicPage] #[AppAPIAuth] @@ -45,20 +37,12 @@ public function registerProvider(string $name, string $displayName, string $acti $exApp = $this->service->getExApp($appId); $provider = $this->speechToTextService->registerSpeechToTextProvider($exApp, $name, $displayName, $actionHandlerRoute); - if ($provider === null) { - throw new OCSBadRequestException('Failed to register STT provider'); + return new DataResponse([], Http::STATUS_BAD_REQUEST); } - return new DataResponse(); } - /** - * @param string $name - * - * @throws OCSBadRequestException - * @return Response - */ #[NoCSRFRequired] #[PublicPage] #[AppAPIAuth] @@ -66,11 +50,9 @@ public function unregisterProvider(string $name): Response { $appId = $this->request->getHeader('EX-APP-ID'); $exApp = $this->service->getExApp($appId); $unregistered = $this->speechToTextService->unregisterSpeechToTextProvider($exApp, $name); - if ($unregistered === null) { - throw new OCSBadRequestException('Failed to unregister STT provider'); + return new DataResponse([], Http::STATUS_BAD_REQUEST); } - return new DataResponse(); } } diff --git a/lib/Db/SpeechToText/ExAppSpeechToTextProvider.php b/lib/Db/SpeechToText/ExAppSpeechToTextProvider.php index bb19c851..2340e5f9 100644 --- a/lib/Db/SpeechToText/ExAppSpeechToTextProvider.php +++ b/lib/Db/SpeechToText/ExAppSpeechToTextProvider.php @@ -14,25 +14,23 @@ * @method string getAppid() * @method string getName() * @method string getDisplayName() - * @method string getDescription() - * @method string getActionHandlerRoute() + * @method string getActionHandler() * @method void setAppid(string $appid) * @method void setName(string $name) * @method void setDisplayName(string $displayName) - * @method void setDescription(string $description) - * @method void setActionHandlerRoute(string $actionHandlerRoute) + * @method void setActionHandler(string $actionHandler) */ class ExAppSpeechToTextProvider extends Entity implements \JsonSerializable { protected $appid; protected $name; protected $displayName; - protected $actionHandlerRoute; + protected $actionHandler; public function __construct(array $params = []) { $this->addType('appid', 'string'); $this->addType('name', 'string'); $this->addType('displayName', 'string'); - $this->addType('actionHandlerRoute', 'string'); + $this->addType('actionHandler', 'string'); if (isset($params['id'])) { $this->setId($params['id']); @@ -46,8 +44,8 @@ public function __construct(array $params = []) { if (isset($params['display_name'])) { $this->setDisplayName($params['display_name']); } - if (isset($params['action_handler_route'])) { - $this->setActionHandlerRoute($params['action_handler_route']); + if (isset($params['action_handler'])) { + $this->setActionHandler($params['action_handler']); } } @@ -57,7 +55,7 @@ public function jsonSerialize(): array { 'appid' => $this->getAppid(), 'name' => $this->getName(), 'display_name' => $this->getDisplayName(), - 'action_handler_route' => $this->getActionHandlerRoute(), + 'action_handler' => $this->getActionHandler(), ]; } } diff --git a/lib/Migration/Version1005Date202312271744.php b/lib/Migration/Version1005Date202312271744.php index 4a43a8a8..d9f3c8c4 100644 --- a/lib/Migration/Version1005Date202312271744.php +++ b/lib/Migration/Version1005Date202312271744.php @@ -22,8 +22,8 @@ public function changeSchema(IOutput $output, Closure $schemaClosure, array $opt /** @var ISchemaWrapper $schema */ $schema = $schemaClosure(); - if (!$schema->hasTable('ex_apps_speech_to_text')) { - $table = $schema->createTable('ex_apps_speech_to_text'); + if (!$schema->hasTable('ex_speech_to_text')) { + $table = $schema->createTable('ex_speech_to_text'); $table->addColumn('id', Types::BIGINT, [ 'autoincrement' => true, @@ -41,14 +41,13 @@ public function changeSchema(IOutput $output, Closure $schemaClosure, array $opt 'notnull' => true, 'length' => 64, ]); - // ExApp route to forward the action - $table->addColumn('action_handler_route', Types::STRING, [ + $table->addColumn('action_handler', Types::STRING, [ 'notnull' => true, - 'length' => 64, + 'length' => 410, ]); - $table->setPrimaryKey(['id'], 'ex_apps_speech_to_text_id'); - $table->addUniqueIndex(['appid', 'name'], 'speech_to_text_appid_name'); + $table->setPrimaryKey(['id']); + $table->addUniqueIndex(['appid', 'name'], 'speech_to_text__idx'); } return $schema; From 219189e18a834ed6774d654d00b733a69251e99d Mon Sep 17 00:00:00 2001 From: Alexander Piskun Date: Fri, 29 Dec 2023 14:26:27 +0300 Subject: [PATCH 06/11] work preparations --- lib/Controller/SpeechToTextController.php | 14 +- ...tProvider.php => SpeechToTextProvider.php} | 2 +- ...per.php => SpeechToTextProviderMapper.php} | 54 +++--- lib/Service/AppAPIService.php | 8 +- lib/Service/SpeechToTextService.php | 162 +++++++++++------- 5 files changed, 139 insertions(+), 101 deletions(-) rename lib/Db/SpeechToText/{ExAppSpeechToTextProvider.php => SpeechToTextProvider.php} (94%) rename lib/Db/SpeechToText/{ExAppSpeechToTextProviderMapper.php => SpeechToTextProviderMapper.php} (51%) diff --git a/lib/Controller/SpeechToTextController.php b/lib/Controller/SpeechToTextController.php index 4b72704a..18ae7e11 100644 --- a/lib/Controller/SpeechToTextController.php +++ b/lib/Controller/SpeechToTextController.php @@ -6,13 +6,11 @@ use OCA\AppAPI\AppInfo\Application; use OCA\AppAPI\Attribute\AppAPIAuth; -use OCA\AppAPI\Service\AppAPIService; use OCA\AppAPI\Service\SpeechToTextService; use OCP\AppFramework\Http; use OCP\AppFramework\Http\Attribute\NoCSRFRequired; use OCP\AppFramework\Http\Attribute\PublicPage; use OCP\AppFramework\Http\DataResponse; -use OCP\AppFramework\Http\Response; use OCP\AppFramework\OCSController; use OCP\IRequest; @@ -21,7 +19,6 @@ class SpeechToTextController extends OCSController { public function __construct( IRequest $request, - private readonly AppAPIService $service, private readonly SpeechToTextService $speechToTextService, ) { parent::__construct(Application::APP_ID, $request); @@ -32,11 +29,9 @@ public function __construct( #[NoCSRFRequired] #[PublicPage] #[AppAPIAuth] - public function registerProvider(string $name, string $displayName, string $actionHandlerRoute): Response { + public function registerProvider(string $name, string $displayName, string $actionHandler): DataResponse { $appId = $this->request->getHeader('EX-APP-ID'); - $exApp = $this->service->getExApp($appId); - - $provider = $this->speechToTextService->registerSpeechToTextProvider($exApp, $name, $displayName, $actionHandlerRoute); + $provider = $this->speechToTextService->registerSpeechToTextProvider($appId, $name, $displayName, $actionHandler); if ($provider === null) { return new DataResponse([], Http::STATUS_BAD_REQUEST); } @@ -46,10 +41,9 @@ public function registerProvider(string $name, string $displayName, string $acti #[NoCSRFRequired] #[PublicPage] #[AppAPIAuth] - public function unregisterProvider(string $name): Response { + public function unregisterProvider(string $name): DataResponse { $appId = $this->request->getHeader('EX-APP-ID'); - $exApp = $this->service->getExApp($appId); - $unregistered = $this->speechToTextService->unregisterSpeechToTextProvider($exApp, $name); + $unregistered = $this->speechToTextService->unregisterSpeechToTextProvider($appId, $name); if ($unregistered === null) { return new DataResponse([], Http::STATUS_BAD_REQUEST); } diff --git a/lib/Db/SpeechToText/ExAppSpeechToTextProvider.php b/lib/Db/SpeechToText/SpeechToTextProvider.php similarity index 94% rename from lib/Db/SpeechToText/ExAppSpeechToTextProvider.php rename to lib/Db/SpeechToText/SpeechToTextProvider.php index 2340e5f9..2b997a64 100644 --- a/lib/Db/SpeechToText/ExAppSpeechToTextProvider.php +++ b/lib/Db/SpeechToText/SpeechToTextProvider.php @@ -20,7 +20,7 @@ * @method void setDisplayName(string $displayName) * @method void setActionHandler(string $actionHandler) */ -class ExAppSpeechToTextProvider extends Entity implements \JsonSerializable { +class SpeechToTextProvider extends Entity implements \JsonSerializable { protected $appid; protected $name; protected $displayName; diff --git a/lib/Db/SpeechToText/ExAppSpeechToTextProviderMapper.php b/lib/Db/SpeechToText/SpeechToTextProviderMapper.php similarity index 51% rename from lib/Db/SpeechToText/ExAppSpeechToTextProviderMapper.php rename to lib/Db/SpeechToText/SpeechToTextProviderMapper.php index 01802eeb..3555b80a 100644 --- a/lib/Db/SpeechToText/ExAppSpeechToTextProviderMapper.php +++ b/lib/Db/SpeechToText/SpeechToTextProviderMapper.php @@ -12,9 +12,9 @@ use OCP\IDBConnection; /** - * @template-extends QBMapper + * @template-extends QBMapper */ -class ExAppSpeechToTextProviderMapper extends QBMapper { +class SpeechToTextProviderMapper extends QBMapper { public function __construct(IDBConnection $db) { parent::__construct($db, 'ex_apps_speech_to_text'); } @@ -22,45 +22,51 @@ public function __construct(IDBConnection $db) { /** * @throws Exception */ - public function findAll(int $limit = null, int $offset = null): array { + public function findAllEnabled(): array { $qb = $this->db->getQueryBuilder(); - $qb->select('*') - ->from($this->tableName) - ->setMaxResults($limit) - ->setFirstResult($offset); - return $this->findEntities($qb); + $result = $qb->select( + 'ex_apps_speech_to_text.appid', + 'ex_apps_speech_to_text.name', + 'ex_apps_speech_to_text.display_name', + 'ex_apps_speech_to_text.action_handler', + ) + ->from($this->tableName, 'ex_apps_speech_to_text') + ->innerJoin('ex_apps_speech_to_text', 'ex_apps', 'exa', 'exa.appid = ex_apps_speech_to_text.appid') + ->where( + $qb->expr()->eq('exa.enabled', $qb->createNamedParameter(1, IQueryBuilder::PARAM_INT)) + ) + ->executeQuery(); + return $result->fetchAll(); } /** * @param string $appId + * @param string $name * + * @return SpeechToTextProvider * @throws Exception - * @return ExAppSpeechToTextProvider[] + * @throws MultipleObjectsReturnedException + * + * @throws DoesNotExistException */ - public function findByAppid(string $appId): array { + public function findByAppidName(string $appId, string $name): SpeechToTextProvider { $qb = $this->db->getQueryBuilder(); - return $this->findEntities($qb->select('*') + return $this->findEntity($qb->select('*') ->from($this->tableName) ->where($qb->expr()->eq('appid', $qb->createNamedParameter($appId), IQueryBuilder::PARAM_STR)) + ->andWhere($qb->expr()->eq('name', $qb->createNamedParameter($name), IQueryBuilder::PARAM_STR)) ); } /** - * @param string $appId - * @param string $name - * - * @throws DoesNotExistException * @throws Exception - * @throws MultipleObjectsReturnedException - * - * @return ExAppSpeechToTextProvider */ - public function findByAppidName(string $appId, string $name): ExAppSpeechToTextProvider { + public function removeAllByAppId(string $appId): int { $qb = $this->db->getQueryBuilder(); - return $this->findEntity($qb->select('*') - ->from($this->tableName) - ->where($qb->expr()->eq('appid', $qb->createNamedParameter($appId), IQueryBuilder::PARAM_STR)) - ->andWhere($qb->expr()->eq('name', $qb->createNamedParameter($name), IQueryBuilder::PARAM_STR)) - ); + $qb->delete($this->tableName) + ->where( + $qb->expr()->eq('appid', $qb->createNamedParameter($appId, IQueryBuilder::PARAM_STR)) + ); + return $qb->executeStatement(); } } diff --git a/lib/Service/AppAPIService.php b/lib/Service/AppAPIService.php index e4206560..d912d2d3 100644 --- a/lib/Service/AppAPIService.php +++ b/lib/Service/AppAPIService.php @@ -45,9 +45,9 @@ class AppAPIService { private IClient $client; public function __construct( - private readonly LoggerInterface $logger, - private readonly ILogFactory $logFactory, - ICacheFactory $cacheFactory, + private readonly LoggerInterface $logger, + private readonly ILogFactory $logFactory, + ICacheFactory $cacheFactory, private readonly IThrottler $throttler, private readonly IConfig $config, IClientService $clientService, @@ -61,6 +61,7 @@ public function __construct( private readonly ScriptsService $scriptsService, private readonly StylesService $stylesService, private readonly FilesActionsMenuService $filesActionsMenuService, + private readonly SpeechToTextService $speechToTextService, private readonly ISecureRandom $random, private readonly IUserSession $userSession, private readonly ISession $session, @@ -149,6 +150,7 @@ public function unregisterExApp(string $appId): ?ExApp { $this->initialStateService->deleteExAppInitialStates($appId); $this->scriptsService->deleteExAppScripts($appId); $this->stylesService->deleteExAppStyles($appId); + $this->speechToTextService->unregisterExAppSpeechToTextProviders($appId); $this->cache->remove('/exApp_' . $appId); return $exApp; } catch (Exception $e) { diff --git a/lib/Service/SpeechToTextService.php b/lib/Service/SpeechToTextService.php index c5520e4e..d274eead 100644 --- a/lib/Service/SpeechToTextService.php +++ b/lib/Service/SpeechToTextService.php @@ -5,10 +5,11 @@ namespace OCA\AppAPI\Service; use OCA\AppAPI\AppInfo\Application; -use OCA\AppAPI\Db\ExApp; -use OCA\AppAPI\Db\SpeechToText\ExAppSpeechToTextProvider; -use OCA\AppAPI\Db\SpeechToText\ExAppSpeechToTextProviderMapper; +use OCA\AppAPI\Db\SpeechToText\SpeechToTextProvider; +use OCA\AppAPI\Db\SpeechToText\SpeechToTextProviderMapper; use OCP\AppFramework\Bootstrap\IRegistrationContext; +use OCP\AppFramework\Db\DoesNotExistException; +use OCP\AppFramework\Db\MultipleObjectsReturnedException; use OCP\AppFramework\Http; use OCP\DB\Exception; use OCP\Files\File; @@ -22,83 +23,118 @@ class SpeechToTextService { private ICache $cache; public function __construct( - ICacheFactory $cacheFactory, - private readonly ExAppSpeechToTextProviderMapper $speechToTextProviderMapper, - private readonly LoggerInterface $logger, - private readonly ?string $userId, + ICacheFactory $cacheFactory, + private readonly SpeechToTextProviderMapper $mapper, + private readonly LoggerInterface $logger, + private readonly ?string $userId, ) { - $this->cache = $cacheFactory->createDistributed(Application::APP_ID . '/ex_apps_speech_to_text'); + $this->cache = $cacheFactory->createDistributed(Application::APP_ID . '/ex_speech_to_text_providers'); } - public function getSpeechToTextProviders(): array { - $cacheKey = '/ex_app_speech_to_text_providers'; - $cached = $this->cache->get($cacheKey); - if ($cached !== null) { - return array_map(function ($cachedEntry) { - return $cachedEntry instanceof ExAppSpeechToTextProvider ? $cachedEntry : new ExAppSpeechToTextProvider($cachedEntry); - }, $cached); + public function registerSpeechToTextProvider(string $appId, string $name, string $displayName, string $actionHandler): ?SpeechToTextProvider { + try { + $speechToTextProvider = $this->mapper->findByAppidName($appId, $name); + } catch (DoesNotExistException|MultipleObjectsReturnedException|Exception) { + $speechToTextProvider = null; } - - $providers = $this->speechToTextProviderMapper->findAll(); - $this->cache->set($cacheKey, $providers); - return $providers; - } - - public function getExAppSpeechToTextProvider(ExApp $exApp, string $name): ?ExAppSpeechToTextProvider { - $cacheKey = '/ex_app_speech_to_text_providers/' . $exApp->getAppid() . '/' . $name; - $cached = $this->cache->get($cacheKey); - if ($cached !== null) { - return $cached instanceof ExAppSpeechToTextProvider ? $cached : new ExAppSpeechToTextProvider($cached); + try { + $newSpeechToTextProvider = new SpeechToTextProvider([ + 'appid' => $appId, + 'name' => $name, + 'display_name' => $displayName, + 'action_handler' => ltrim($actionHandler, '/'), + ]); + if ($speechToTextProvider !== null) { + $speechToTextProvider->setId($speechToTextProvider->getId()); + } + $speechToTextProvider = $this->mapper->insertOrUpdate($newSpeechToTextProvider); + $this->cache->set('/ex_speech_to_text_providers_' . $appId . '_' . $name, $speechToTextProvider); + $this->resetCacheEnabled(); + } catch (Exception $e) { + $this->logger->error( + sprintf('Failed to register ExApp %s SpeechToTextProvider %s. Error: %s', $appId, $name, $e->getMessage()), ['exception' => $e] + ); + return null; } - - $provider = $this->speechToTextProviderMapper->findByAppidName($exApp->getAppid(), $name); - $this->cache->set($cacheKey, $provider); - return $provider; + return $speechToTextProvider; } - public function registerSpeechToTextProvider(ExApp $exApp, string $name, string $displayName, string $actionHandlerRoute): ?ExAppSpeechToTextProvider { - $provider = new ExAppSpeechToTextProvider([ - 'appid' => $exApp->getAppid(), - 'name' => $name, - 'display_name' => $displayName, - 'action_handler_route' => $actionHandlerRoute, - ]); + public function unregisterSpeechToTextProvider(string $appId, string $name): ?SpeechToTextProvider { try { - $this->speechToTextProviderMapper->insert($provider); - $this->cache->remove('/ex_app_speech_to_text_providers'); - return $provider; + $speechToTextProvider = $this->getExAppSpeechToTextProvider($appId, $name); + if ($speechToTextProvider === null) { + return null; + } + $this->mapper->delete($speechToTextProvider); + $this->cache->remove('/ex_speech_to_text_providers_' . $appId . '_' . $name); + $this->resetCacheEnabled(); + return $speechToTextProvider; } catch (Exception $e) { - $this->logger->error('Failed to register SpeechToText provider', ['exception' => $e]); + $this->logger->error(sprintf('Failed to unregister ExApp %s SpeechToTextProvider %s. Error: %s', $appId, $name, $e->getMessage()), ['exception' => $e]); return null; } } - public function unregisterSpeechToTextProvider(ExApp $exApp, string $name): ?ExAppSpeechToTextProvider { - $provider = $this->getExAppSpeechToTextProvider($exApp, $name); - if ($provider === null) { - return null; + /** + * Get list of registered file actions (only for enabled ExApps) + * + * @return SpeechToTextProvider[] + */ + public function getRegisteredSpeechToTextProviders(): array { + try { + $cacheKey = '/ex_speech_to_text_providers'; + $cached = $this->cache->get($cacheKey); + if ($cached !== null) { + return array_map(function ($cacheEntry) { + return $cacheEntry instanceof SpeechToTextProvider ? $cacheEntry : new SpeechToTextProvider($cacheEntry); + }, $cached); + } + + $speechToTextProviders = $this->mapper->findAllEnabled(); + $this->cache->set($cacheKey, $speechToTextProviders); + return $speechToTextProviders; + } catch (Exception) { + return []; + } + } + + public function getExAppSpeechToTextProvider(string $appId, string $name): ?SpeechToTextProvider { + $cacheKey = '/ex_speech_to_text_providers_' . $appId . '_' . $name; + $cache = $this->cache->get($cacheKey); + if ($cache !== null) { + return $cache instanceof SpeechToTextProvider ? $cache : new SpeechToTextProvider($cache); } + try { - $this->speechToTextProviderMapper->delete($provider); - $this->cache->remove('/ex_app_speech_to_text_providers'); - return $provider; - } catch (Exception $e) { - $this->logger->error('Failed to unregister STT provider', ['exception' => $e]); + $speechToTextProvider = $this->mapper->findByAppIdName($appId, $name); + } catch (DoesNotExistException|MultipleObjectsReturnedException|Exception) { return null; } + $this->cache->set($cacheKey, $speechToTextProvider); + return $speechToTextProvider; + } + + public function unregisterExAppSpeechToTextProviders(string $appId): int { + try { + $result = $this->mapper->removeAllByAppId($appId); + } catch (Exception) { + $result = -1; + } + $this->cache->clear('/ex_speech_to_text_providers_' . $appId); + $this->resetCacheEnabled(); + return $result; + } + + public function resetCacheEnabled(): void { + $this->cache->remove('/ex_speech_to_text_providers'); } /** * Register ExApp anonymous providers implementations of ISpeechToTextProviderWithId - * so that they can be used as regular providers in DI container - * - * @param IRegistrationContext $context - * - * @return void + * so that they can be used as regular providers in DI container. */ public function registerExAppSpeechToTextProviders(IRegistrationContext &$context, IServerContainer $serverContainer): void { - $exAppsProviders = $this->getSpeechToTextProviders(); - /** @var ExAppSpeechToTextProvider $exAppProvider */ + $exAppsProviders = $this->getRegisteredSpeechToTextProviders(); foreach ($exAppsProviders as $exAppProvider) { $class = '\\OCA\\AppAPI\\' . $exAppProvider->getAppid() . '\\' . $exAppProvider->getName(); $sttProvider = $this->getAnonymousExAppProvider($exAppProvider, $serverContainer, $class); @@ -112,14 +148,14 @@ public function registerExAppSpeechToTextProviders(IRegistrationContext &$contex /** * @psalm-suppress UndefinedClass, MissingDependency, InvalidReturnStatement, InvalidReturnType */ - private function getAnonymousExAppProvider(ExAppSpeechToTextProvider $provider, IServerContainer $serverContainer, string $class): ?ISpeechToTextProviderWithId { + private function getAnonymousExAppProvider(SpeechToTextProvider $provider, IServerContainer $serverContainer, string $class): ?ISpeechToTextProviderWithId { return new class($provider, $serverContainer, $this->userId, $class) implements ISpeechToTextProviderWithId { public function __construct( - private ExAppSpeechToTextProvider $sttProvider, + private SpeechToTextProvider $sttProvider, // We need this to delay the instantiation of AppAPIService during registration to avoid conflicts - private IServerContainer $serverContainer, // TODO: Extract needed methods from AppAPIService to be able to use it everytime - private readonly ?string $userId, - private readonly string $class, + private IServerContainer $serverContainer, // TODO: Extract needed methods from AppAPIService to be able to use it everytime + private readonly ?string $userId, + private readonly string $class, ) { } @@ -132,7 +168,7 @@ public function getName(): string { } public function transcribeFile(File $file): string { - $route = $this->sttProvider->getActionHandlerRoute(); + $route = $this->sttProvider->getActionHandler(); $service = $this->serverContainer->get(AppAPIService::class); $exApp = $service->getExApp($this->sttProvider->getAppid()); From 73f289d2d83e1ddd3c6383bf898e5ccfac8f27ce Mon Sep 17 00:00:00 2001 From: Alexander Piskun Date: Fri, 29 Dec 2023 19:06:36 +0300 Subject: [PATCH 07/11] added scope, rewrote "transcribeFile" --- appinfo/routes.php | 4 +-- docs/tech_details/ApiScopes.rst | 1 + lib/Controller/SpeechToTextController.php | 12 ++++++- .../SpeechToTextProviderMapper.php | 14 ++++---- lib/Service/ExAppApiScopeService.php | 1 + lib/Service/SpeechToTextService.php | 34 +++++++++++++++---- 6 files changed, 50 insertions(+), 16 deletions(-) diff --git a/appinfo/routes.php b/appinfo/routes.php index 3a4e90ca..f6c1c47c 100644 --- a/appinfo/routes.php +++ b/appinfo/routes.php @@ -102,7 +102,7 @@ ['name' => 'OCSUi#getExAppStyle', 'url' => '/api/v1/ui/style', 'verb' => 'GET'], // Speech-To-Text - ['name' => 'speechToText#registerProvider', 'url' => '/api/v1/provider/speech_to_text', 'verb' => 'POST'], - ['name' => 'speechToText#unregisterProvider', 'url' => '/api/v1/provider/speech_to_text', 'verb' => 'DELETE'], + ['name' => 'speechToText#registerProvider', 'url' => '/api/v1/ai_provider/speech_to_text', 'verb' => 'POST'], + ['name' => 'speechToText#unregisterProvider', 'url' => '/api/v1/ai_provider/speech_to_text', 'verb' => 'DELETE'], ], ]; diff --git a/docs/tech_details/ApiScopes.rst b/docs/tech_details/ApiScopes.rst index 2dc815f9..f915dd7f 100644 --- a/docs/tech_details/ApiScopes.rst +++ b/docs/tech_details/ApiScopes.rst @@ -29,6 +29,7 @@ The following API groups are currently supported: * ``33`` WEATHER_STATUS * ``50`` TALK * ``60`` TALK_BOT +* ``61`` AI_PROVIDERS * ``110`` ACTIVITIES * ``120`` NOTES diff --git a/lib/Controller/SpeechToTextController.php b/lib/Controller/SpeechToTextController.php index 18ae7e11..5532c297 100644 --- a/lib/Controller/SpeechToTextController.php +++ b/lib/Controller/SpeechToTextController.php @@ -12,6 +12,7 @@ use OCP\AppFramework\Http\Attribute\PublicPage; use OCP\AppFramework\Http\DataResponse; use OCP\AppFramework\OCSController; +use OCP\IConfig; use OCP\IRequest; class SpeechToTextController extends OCSController { @@ -20,6 +21,7 @@ class SpeechToTextController extends OCSController { public function __construct( IRequest $request, private readonly SpeechToTextService $speechToTextService, + private readonly IConfig $config, ) { parent::__construct(Application::APP_ID, $request); @@ -30,6 +32,10 @@ public function __construct( #[PublicPage] #[AppAPIAuth] public function registerProvider(string $name, string $displayName, string $actionHandler): DataResponse { + $ncVersion = $this->config->getSystemValueString('version', '0.0.0'); + if (version_compare($ncVersion, '29.0', '<')) { + return new DataResponse([], Http::STATUS_NOT_IMPLEMENTED); + } $appId = $this->request->getHeader('EX-APP-ID'); $provider = $this->speechToTextService->registerSpeechToTextProvider($appId, $name, $displayName, $actionHandler); if ($provider === null) { @@ -42,10 +48,14 @@ public function registerProvider(string $name, string $displayName, string $acti #[PublicPage] #[AppAPIAuth] public function unregisterProvider(string $name): DataResponse { + $ncVersion = $this->config->getSystemValueString('version', '0.0.0'); + if (version_compare($ncVersion, '29.0', '<')) { + return new DataResponse([], Http::STATUS_NOT_IMPLEMENTED); + } $appId = $this->request->getHeader('EX-APP-ID'); $unregistered = $this->speechToTextService->unregisterSpeechToTextProvider($appId, $name); if ($unregistered === null) { - return new DataResponse([], Http::STATUS_BAD_REQUEST); + return new DataResponse([], Http::STATUS_NOT_FOUND); } return new DataResponse(); } diff --git a/lib/Db/SpeechToText/SpeechToTextProviderMapper.php b/lib/Db/SpeechToText/SpeechToTextProviderMapper.php index 3555b80a..98a4ac8e 100644 --- a/lib/Db/SpeechToText/SpeechToTextProviderMapper.php +++ b/lib/Db/SpeechToText/SpeechToTextProviderMapper.php @@ -16,7 +16,7 @@ */ class SpeechToTextProviderMapper extends QBMapper { public function __construct(IDBConnection $db) { - parent::__construct($db, 'ex_apps_speech_to_text'); + parent::__construct($db, 'ex_speech_to_text'); } /** @@ -25,13 +25,13 @@ public function __construct(IDBConnection $db) { public function findAllEnabled(): array { $qb = $this->db->getQueryBuilder(); $result = $qb->select( - 'ex_apps_speech_to_text.appid', - 'ex_apps_speech_to_text.name', - 'ex_apps_speech_to_text.display_name', - 'ex_apps_speech_to_text.action_handler', + 'ex_speech_to_text.appid', + 'ex_speech_to_text.name', + 'ex_speech_to_text.display_name', + 'ex_speech_to_text.action_handler', ) - ->from($this->tableName, 'ex_apps_speech_to_text') - ->innerJoin('ex_apps_speech_to_text', 'ex_apps', 'exa', 'exa.appid = ex_apps_speech_to_text.appid') + ->from($this->tableName, 'ex_speech_to_text') + ->innerJoin('ex_speech_to_text', 'ex_apps', 'exa', 'exa.appid = ex_speech_to_text.appid') ->where( $qb->expr()->eq('exa.enabled', $qb->createNamedParameter(1, IQueryBuilder::PARAM_INT)) ) diff --git a/lib/Service/ExAppApiScopeService.php b/lib/Service/ExAppApiScopeService.php index f7d99187..b38b990f 100644 --- a/lib/Service/ExAppApiScopeService.php +++ b/lib/Service/ExAppApiScopeService.php @@ -95,6 +95,7 @@ public function registerInitScopes(): bool { ['api_route' => $aeApiV1Prefix . '/ex-app/enabled', 'scope_group' => 2, 'name' => 'SYSTEM', 'user_check' => 1], ['api_route' => $aeApiV1Prefix . '/notification', 'scope_group' => 32, 'name' => 'NOTIFICATIONS', 'user_check' => 1], ['api_route' => $aeApiV1Prefix . '/talk_bot', 'scope_group' => 60, 'name' => 'TALK_BOT', 'user_check' => 0], + ['api_route' => $aeApiV1Prefix . '/ai_provider/', 'scope_group' => 61, 'name' => 'AI_PROVIDERS', 'user_check' => 0], // AppAPI internal scopes ['api_route' => '/apps/app_api/apps/status', 'scope_group' => 1, 'name' => 'BASIC', 'user_check' => 0], diff --git a/lib/Service/SpeechToTextService.php b/lib/Service/SpeechToTextService.php index d274eead..35b3bcd4 100644 --- a/lib/Service/SpeechToTextService.php +++ b/lib/Service/SpeechToTextService.php @@ -170,16 +170,38 @@ public function getName(): string { public function transcribeFile(File $file): string { $route = $this->sttProvider->getActionHandler(); $service = $this->serverContainer->get(AppAPIService::class); - $exApp = $service->getExApp($this->sttProvider->getAppid()); - $response = $service->requestToExApp($exApp, $route, $this->userId, 'POST', [ - 'fileid' => $file->getId(), - ]); + try { + $fileHandle = $file->fopen('r'); + $response = $service->requestToExAppById($this->sttProvider->getAppid(), + $route, + $this->userId, + 'POST', + options: [ + 'multipart' => [ + 'name' => 'data', + 'contents' => $fileHandle, + 'filename' => $file->getName(), + 'headers' => [ + 'Content-Type' => $file->getMimeType(), + ] + ], + ]); + } catch (Exception $e) { + $this->logger->error( + sprintf('Failed to transcribe file: %s with %s:%s. Error: %s', + $file->getName(), + $this->sttProvider->getAppid(), + $this->sttProvider->getName(), + $e->getMessage() + ), ['exception' => $e] + ); + return ''; + } if ($response->getStatusCode() !== Http::STATUS_OK) { - throw new \Exception('Failed to transcribe file'); + throw new \Exception(sprintf('Failed to transcribe file, status: %s.', $response->getStatusCode())); } - return $response->getBody(); } }; From 897bdbd7b196025b11f2115d0d891e7aaad7de55 Mon Sep 17 00:00:00 2001 From: Alexander Piskun Date: Fri, 29 Dec 2023 22:28:25 +0300 Subject: [PATCH 08/11] first working version --- lib/Db/SpeechToText/SpeechToTextProvider.php | 2 +- lib/Service/AppAPIService.php | 7 ++- lib/Service/SpeechToTextService.php | 45 ++++++++++---------- 3 files changed, 30 insertions(+), 24 deletions(-) diff --git a/lib/Db/SpeechToText/SpeechToTextProvider.php b/lib/Db/SpeechToText/SpeechToTextProvider.php index 2b997a64..0c6a1073 100644 --- a/lib/Db/SpeechToText/SpeechToTextProvider.php +++ b/lib/Db/SpeechToText/SpeechToTextProvider.php @@ -9,7 +9,7 @@ /** * Class ExAppSpeechToTextProvider * - * @package OCA\AppAPI\Db + * @package OCA\AppAPI\Db\SpeechToText * * @method string getAppid() * @method string getName() diff --git a/lib/Service/AppAPIService.php b/lib/Service/AppAPIService.php index d912d2d3..e2419219 100644 --- a/lib/Service/AppAPIService.php +++ b/lib/Service/AppAPIService.php @@ -567,7 +567,12 @@ public function requestToExApp( $url = self::getExAppUrl( $exApp->getProtocol(), $exApp->getHost(), - $exApp->getPort()) . $route; + $exApp->getPort()); + if (str_starts_with($route, '/')) { + $url = $url.$route; + } else { + $url = $url.'/'.$route; + } if (isset($options['headers']) && is_array($options['headers'])) { $options['headers'] = [...$options['headers'], ...$this->buildAppAPIAuthHeaders($request, $userId, $exApp)]; diff --git a/lib/Service/SpeechToTextService.php b/lib/Service/SpeechToTextService.php index 35b3bcd4..ce2aa32b 100644 --- a/lib/Service/SpeechToTextService.php +++ b/lib/Service/SpeechToTextService.php @@ -167,18 +167,22 @@ public function getName(): string { return $this->sttProvider->getDisplayName(); } - public function transcribeFile(File $file): string { + public function transcribeFile(File $file, float $maxWaitTime=0): string { $route = $this->sttProvider->getActionHandler(); $service = $this->serverContainer->get(AppAPIService::class); try { $fileHandle = $file->fopen('r'); - $response = $service->requestToExAppById($this->sttProvider->getAppid(), - $route, - $this->userId, - 'POST', - options: [ - 'multipart' => [ + } catch (Exception $e) { + throw new \Exception(sprintf('Failed to open file: %s. Error: %s', $file->getName(), $e->getMessage())); + } + $response = $service->requestToExAppById($this->sttProvider->getAppid(), + $route, + $this->userId, + 'POST', + options: [ + 'multipart' => [ + [ 'name' => 'data', 'contents' => $fileHandle, 'filename' => $file->getName(), @@ -186,21 +190,18 @@ public function transcribeFile(File $file): string { 'Content-Type' => $file->getMimeType(), ] ], - ]); - } catch (Exception $e) { - $this->logger->error( - sprintf('Failed to transcribe file: %s with %s:%s. Error: %s', - $file->getName(), - $this->sttProvider->getAppid(), - $this->sttProvider->getName(), - $e->getMessage() - ), ['exception' => $e] - ); - return ''; - } - - if ($response->getStatusCode() !== Http::STATUS_OK) { - throw new \Exception(sprintf('Failed to transcribe file, status: %s.', $response->getStatusCode())); + ], + 'timeout' => $maxWaitTime, + ]); + if (is_array($response)) { + throw new \Exception(sprintf('Failed to transcribe file: %s with %s:%s. Error: %s', + $file->getName(), + $this->sttProvider->getAppid(), + $this->sttProvider->getName(), + $response['error'] + )); + } else if ($response->getStatusCode() !== Http::STATUS_OK) { + throw new \Exception(sprintf('ExApp failed to transcribe file, status: %s.', $response->getStatusCode())); } return $response->getBody(); } From e3f1645aad85cb7789b6283cdcb6ae87898b8320 Mon Sep 17 00:00:00 2001 From: Alexander Piskun Date: Sun, 31 Dec 2023 13:19:15 +0300 Subject: [PATCH 09/11] added "getProvider" method, fixed "Error during app service registration" --- appinfo/routes.php | 1 + lib/Controller/SpeechToTextController.php | 24 ++++++++++++++++++---- lib/Service/SpeechToTextService.php | 25 ++++++++++------------- 3 files changed, 32 insertions(+), 18 deletions(-) diff --git a/appinfo/routes.php b/appinfo/routes.php index f6c1c47c..9bedeaea 100644 --- a/appinfo/routes.php +++ b/appinfo/routes.php @@ -104,5 +104,6 @@ // Speech-To-Text ['name' => 'speechToText#registerProvider', 'url' => '/api/v1/ai_provider/speech_to_text', 'verb' => 'POST'], ['name' => 'speechToText#unregisterProvider', 'url' => '/api/v1/ai_provider/speech_to_text', 'verb' => 'DELETE'], + ['name' => 'speechToText#getProvider', 'url' => '/api/v1/ai_provider/speech_to_text', 'verb' => 'GET'], ], ]; diff --git a/lib/Controller/SpeechToTextController.php b/lib/Controller/SpeechToTextController.php index 5532c297..112c0011 100644 --- a/lib/Controller/SpeechToTextController.php +++ b/lib/Controller/SpeechToTextController.php @@ -36,8 +36,8 @@ public function registerProvider(string $name, string $displayName, string $acti if (version_compare($ncVersion, '29.0', '<')) { return new DataResponse([], Http::STATUS_NOT_IMPLEMENTED); } - $appId = $this->request->getHeader('EX-APP-ID'); - $provider = $this->speechToTextService->registerSpeechToTextProvider($appId, $name, $displayName, $actionHandler); + $provider = $this->speechToTextService->registerSpeechToTextProvider( + $this->request->getHeader('EX-APP-ID'), $name, $displayName, $actionHandler); if ($provider === null) { return new DataResponse([], Http::STATUS_BAD_REQUEST); } @@ -52,11 +52,27 @@ public function unregisterProvider(string $name): DataResponse { if (version_compare($ncVersion, '29.0', '<')) { return new DataResponse([], Http::STATUS_NOT_IMPLEMENTED); } - $appId = $this->request->getHeader('EX-APP-ID'); - $unregistered = $this->speechToTextService->unregisterSpeechToTextProvider($appId, $name); + $unregistered = $this->speechToTextService->unregisterSpeechToTextProvider( + $this->request->getHeader('EX-APP-ID'), $name); if ($unregistered === null) { return new DataResponse([], Http::STATUS_NOT_FOUND); } return new DataResponse(); } + + #[AppAPIAuth] + #[PublicPage] + #[NoCSRFRequired] + public function getProvider(string $name): DataResponse { + $ncVersion = $this->config->getSystemValueString('version', '0.0.0'); + if (version_compare($ncVersion, '29.0', '<')) { + return new DataResponse([], Http::STATUS_NOT_IMPLEMENTED); + } + $result = $this->speechToTextService->getExAppSpeechToTextProvider( + $this->request->getHeader('EX-APP-ID'), $name); + if (!$result) { + return new DataResponse([], Http::STATUS_NOT_FOUND); + } + return new DataResponse($result, Http::STATUS_OK); + } } diff --git a/lib/Service/SpeechToTextService.php b/lib/Service/SpeechToTextService.php index ce2aa32b..d11da442 100644 --- a/lib/Service/SpeechToTextService.php +++ b/lib/Service/SpeechToTextService.php @@ -10,7 +10,6 @@ use OCP\AppFramework\Bootstrap\IRegistrationContext; use OCP\AppFramework\Db\DoesNotExistException; use OCP\AppFramework\Db\MultipleObjectsReturnedException; -use OCP\AppFramework\Http; use OCP\DB\Exception; use OCP\Files\File; use OCP\ICache; @@ -83,16 +82,14 @@ public function unregisterSpeechToTextProvider(string $appId, string $name): ?Sp public function getRegisteredSpeechToTextProviders(): array { try { $cacheKey = '/ex_speech_to_text_providers'; - $cached = $this->cache->get($cacheKey); - if ($cached !== null) { - return array_map(function ($cacheEntry) { - return $cacheEntry instanceof SpeechToTextProvider ? $cacheEntry : new SpeechToTextProvider($cacheEntry); - }, $cached); + $records = $this->cache->get($cacheKey); + if ($records === null) { + $records = $this->mapper->findAllEnabled(); + $this->cache->set($cacheKey, $records); } - - $speechToTextProviders = $this->mapper->findAllEnabled(); - $this->cache->set($cacheKey, $speechToTextProviders); - return $speechToTextProviders; + return array_map(function ($record) { + return new SpeechToTextProvider($record); + }, $records); } catch (Exception) { return []; } @@ -167,7 +164,7 @@ public function getName(): string { return $this->sttProvider->getDisplayName(); } - public function transcribeFile(File $file, float $maxWaitTime=0): string { + public function transcribeFile(File $file, float $maxExecutionTime = 0): string { $route = $this->sttProvider->getActionHandler(); $service = $this->serverContainer->get(AppAPIService::class); @@ -180,6 +177,7 @@ public function transcribeFile(File $file, float $maxWaitTime=0): string { $route, $this->userId, 'POST', + params: ['max_execution_time' => $$maxExecutionTime], options: [ 'multipart' => [ [ @@ -191,7 +189,8 @@ public function transcribeFile(File $file, float $maxWaitTime=0): string { ] ], ], - 'timeout' => $maxWaitTime, + 'query' => ['max_execution_time' => $maxExecutionTime], + 'timeout' => $maxExecutionTime, ]); if (is_array($response)) { throw new \Exception(sprintf('Failed to transcribe file: %s with %s:%s. Error: %s', @@ -200,8 +199,6 @@ public function transcribeFile(File $file, float $maxWaitTime=0): string { $this->sttProvider->getName(), $response['error'] )); - } else if ($response->getStatusCode() !== Http::STATUS_OK) { - throw new \Exception(sprintf('ExApp failed to transcribe file, status: %s.', $response->getStatusCode())); } return $response->getBody(); } From 8f66609a6115deb510a6d9e5c1802c8530787d15 Mon Sep 17 00:00:00 2001 From: Alexander Piskun Date: Sun, 31 Dec 2023 13:36:31 +0300 Subject: [PATCH 10/11] fixed typo in comment --- lib/Service/SpeechToTextService.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Service/SpeechToTextService.php b/lib/Service/SpeechToTextService.php index d11da442..4d390352 100644 --- a/lib/Service/SpeechToTextService.php +++ b/lib/Service/SpeechToTextService.php @@ -75,7 +75,7 @@ public function unregisterSpeechToTextProvider(string $appId, string $name): ?Sp } /** - * Get list of registered file actions (only for enabled ExApps) + * Get list of registered SpeechToText providers (only for enabled ExApps) * * @return SpeechToTextProvider[] */ From 7082c4f3665b2b40c7a682642b5d81be3f0571a0 Mon Sep 17 00:00:00 2001 From: Alexander Piskun Date: Sun, 31 Dec 2023 15:45:30 +0300 Subject: [PATCH 11/11] fixed bug/typo in "registerFileActionMenu" --- lib/Service/SpeechToTextService.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Service/SpeechToTextService.php b/lib/Service/SpeechToTextService.php index 4d390352..a93a7228 100644 --- a/lib/Service/SpeechToTextService.php +++ b/lib/Service/SpeechToTextService.php @@ -44,7 +44,7 @@ public function registerSpeechToTextProvider(string $appId, string $name, string 'action_handler' => ltrim($actionHandler, '/'), ]); if ($speechToTextProvider !== null) { - $speechToTextProvider->setId($speechToTextProvider->getId()); + $newSpeechToTextProvider->setId($speechToTextProvider->getId()); } $speechToTextProvider = $this->mapper->insertOrUpdate($newSpeechToTextProvider); $this->cache->set('/ex_speech_to_text_providers_' . $appId . '_' . $name, $speechToTextProvider);