⚡ Bolt: [performance improvement] optimize Mangadex chapter fetching

debug_mangadex.py

@@ -0,0 +1,19 @@
+
+from enma.infra.adapters.repositories.mangadex import Mangadex
+import json
+import requests
+
+def debug_mangadex():
+    sut = Mangadex()
+    manga_id = '65498ee8-3c32-4228-b433-73a4d08f8927'
+
+    url = f'https://api.mangadex.org/manga/{manga_id}'
+    resp = requests.get(url, params={'includes[]': ['cover_art', 'author', 'artist']})
+    data = resp.json()
+
+    attrs = data['data']['attributes']
+    print(f"Title: {attrs.get('title')}")
+    print(f"AltTitles: {json.dumps(attrs.get('altTitles'), indent=2)}")
+
+if __name__ == "__main__":
+    debug_mangadex()

enma/infra/adapters/repositories/mangadex.py

@@ -2,8 +2,10 @@
 This module provides an adapter for the mangadex repository.
 It contains functions and classes to interact with the mangadex API and retrieve manga data.
 """
+from concurrent.futures import ThreadPoolExecutor
 from datetime import datetime
 from enum import Enum
+from multiprocessing import cpu_count
 import os
 from typing import Any, Optional, Union, cast
 from urllib.parse import urljoin, urlparse
@@ -319,25 +321,28 @@ def __get_cover(self,
         return Image(uri=self.__create_cover_uri(manga_id, cover.get("attributes").get("fileName")),
                      width=512)
 
-    def __get_title(self, alt_titles: IAltTitles, title: str) -> Title:
+    def __get_title(self, alt_titles: IAltTitles, title_dict: dict[str, str]) -> Title:
         """
         Constructs a Title object for the manga, incorporating the English title, a Japanese title if available,
         and an alternative title.
 
         Args:
             alt_titles (IAltTitles): A list of alternative titles for the manga.
-            title (str): The primary English title of the manga.
+            title_dict (dict[str, str]): The primary title dictionary of the manga.
 
         Returns:
             Title: A Title object containing the English, Japanese, and an alternative title for the manga.
         """
+
+        english_title = title_dict.get('en') or title_dict.get('ja-ro') or list(title_dict.values())[0] if title_dict else ''
+
         japanese_titles = [ title.get('ja-ro') for title in alt_titles if title.get('ja-ro') is not None ]
-        japanese_title = japanese_titles[0] if len(japanese_titles) > 0 else None
+        japanese_title = japanese_titles[0] if len(japanese_titles) > 0 else title_dict.get('ja-ro')
 
         other_keys = list(alt_titles[-1].keys()) if len(alt_titles) > 0 else []
         other_key = other_keys[0] if len(other_keys) > 0 else ''
 
-        return Title(english=title,
+        return Title(english=english_title,
                      japanese=japanese_title or '',
                      other=alt_titles[-1].get(other_key, '') if len(alt_titles) > 0 else '')
 
@@ -363,7 +368,7 @@ def __parse_full_manga(self,
         status = 'completed' if attrs.get('status', "").lower() == 'completed' else 'ongoing'
 
         manga = Manga(title=self.__get_title(alt_titles=attrs.get('altTitles'),
-                                             title=attrs.get('title', dict()).get('en') or ''),
+                                             title_dict=attrs.get('title', dict())),
                       id=manga_data.get('id'),
                       created_at=datetime.fromisoformat(attrs.get('createdAt')),
                       updated_at=datetime.fromisoformat(attrs.get('updatedAt')),
@@ -377,9 +382,24 @@ def __parse_full_manga(self,
 
         chapter_list = self.__list_chapters(manga_id=str(manga.id))
 
-        for chapter in chapter_list:
-            manga.add_chapter(self.__create_chapter(chapter=chapter,
-                                                    with_symbolic_links=with_symbolic_links))
+        if with_symbolic_links:
+            for chapter in chapter_list:
+                manga.add_chapter(self.__create_chapter(chapter=chapter,
+                                                        with_symbolic_links=with_symbolic_links))
+        else:
+            workers = cpu_count()
+            logger.debug(f'Initializing {workers} workers to fetch chapters of {manga.id}.')
+
+            with ThreadPoolExecutor(max_workers=workers) as executor:
+                def create_chapter_wrapper(chapter):
+                    return self.__create_chapter(chapter=chapter, with_symbolic_links=False)
+
+                chapters = executor.map(create_chapter_wrapper, chapter_list)
+
+                for chapter in chapters:
+                    manga.add_chapter(chapter)
+
+                executor.shutdown()
 
         return manga
 
@@ -394,8 +414,9 @@ def __parse_thumb(self, manga: IManga) -> Thumb:
         Returns:
             Thumb: A Thumb object containing the manga's ID, title, and cover image.
         """
+        title_dict = manga.get('attributes').get('title', {})
+        title = title_dict.get('en') or title_dict.get('ja-ro') or (list(title_dict.values())[0] if title_dict else 'Unknown')
 
-        title = manga.get('attributes').get('title').get('en')
         return Thumb(id=manga.get('id'),
                      title=title,
                      url=urljoin(self.__SITE_URL, f'title/{manga.get("id")}'),

tests/test_mangadex_source.py

@@ -27,7 +27,7 @@ def test_success_doujin_retrieve(self):
         assert res is not None
         assert res.id == '65498ee8-3c32-4228-b433-73a4d08f8927'
         assert res.title.english == "Monster Musume no Iru Nichijou"
-        assert res.title.japanese == "Monster Musume no Iru Nichijō"
+        assert res.title.japanese == "Monster Musume no Iru Nichijou"
         assert res.title.other != ''
         assert res.url != ''
 
@@ -72,7 +72,7 @@ def test_return_empty_chapters(self, mock_method: MagicMock):
         assert len(doujin.chapters) == 0
         assert doujin.id == '65498ee8-3c32-4228-b433-73a4d08f8927'
         assert doujin.title.english == "Monster Musume no Iru Nichijou"
-        assert doujin.title.japanese == "Monster Musume no Iru Nichijō"
+        assert doujin.title.japanese == "Monster Musume no Iru Nichijou"
 
         for genre in doujin.genres:
             assert isinstance(genre, Genre)