feat: add Firecrawl scraping feature

.gitignore

@@ -256,3 +256,4 @@ conf/*
 !conf/.gitkeep
 .run/Template Python tests.run.xml
 /.run/
+.DS_Store

app/config.py

@@ -211,6 +211,12 @@ def ban_list_resolver(ban_list_string: str) -> list:
 # Open AI API environment variables
 OPENAI_API_KEY = env.get("OPENAI_API_KEY", None)
 
+# Firecrawl API environment variables
+FIRECRAWL_ON = get_env_bool(env, "FIRECRAWL_ON", False)
+FIRECRAWL_API_URL = os.getenv("FIRECRAWL_API_URL", "")
+FIRECRAWL_API_KEY = os.getenv("FIRECRAWL_API_KEY", "")
+FIRECRAWL_TIMEOUT_SECONDS = env.get("FIRECRAWL_TIMEOUT_SECONDS", 60)
+
 # Locale environment variables
 localedir = os.path.join(os.path.dirname(__file__), "locale")
 translation = gettext.translation("messages", localedir=localedir, fallback=True)

app/services/scrapers/common.py

@@ -8,8 +8,7 @@
     inoreader
 )
 from app.services.file_export import video_download, document_export
-from app.services.scrapers import twitter, wechat, reddit, weibo, zhihu, douban, instagram, xiaohongshu, threads, \
-    bluesky
+from app.services.scrapers import twitter, wechat, reddit, weibo, zhihu, douban, instagram, xiaohongshu, threads
 from app.services.scrapers.scraper_manager import ScraperManager
 from app.database import save_instances
 from app.utils.logger import logger
@@ -61,7 +60,7 @@ async def get_item(self, metadata_item: Optional[dict] = None) -> dict:
                 self.kwargs["category"] = self.category
         if not metadata_item:
             try:
-                if self.category in ["bluesky", "weibo"]:  # it is a workaround before the code refactor
+                if self.category in ["bluesky", "weibo", "other", "unknown"]:  # it is a workaround before the code refactor
                     await ScraperManager.init_scraper(self.category)
                     item_data_processor = await ScraperManager.scrapers[self.category].get_processor_by_url(url=self.url)
                     metadata_item = await item_data_processor.get_item()

app/services/scrapers/firecrawl_client/__init__.py

@@ -0,0 +1,37 @@
+from dataclasses import dataclass
+from typing import Any
+
+from app.models.metadata_item import MetadataItem
+
+
+@dataclass
+class FirecrawlItem(MetadataItem):
+    """
+    FirecrawlItem: Data class for scraped content from Firecrawl.
+    """
+    id: str = ""
+    raw_content: str = ""
+
+    @staticmethod
+    def from_dict(obj: Any) -> "FirecrawlItem":
+        metadata_item = MetadataItem.from_dict(obj)
+        return FirecrawlItem(
+            url=metadata_item.url,
+            title=metadata_item.title,
+            author=metadata_item.author,
+            author_url=metadata_item.author_url,
+            telegraph_url=metadata_item.telegraph_url,
+            text=metadata_item.text,
+            content=metadata_item.content,
+            media_files=metadata_item.media_files,
+            category=metadata_item.category,
+            message_type=metadata_item.message_type,
+            id=obj.get("id", ""),
+            raw_content=obj.get("raw_content", ""),
+        )
+
+    def to_dict(self) -> dict:
+        result: dict = super().to_dict()
+        result["id"] = self.id
+        result["raw_content"] = self.raw_content
+        return result

app/services/scrapers/firecrawl_client/client.py

@@ -0,0 +1,95 @@
+from __future__ import annotations
+
+import threading
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional
+
+from firecrawl import Firecrawl
+
+from app.config import FIRECRAWL_API_URL, FIRECRAWL_API_KEY, FIRECRAWL_TIMEOUT_SECONDS
+
+
+@dataclass(frozen=True)
+class FirecrawlSettings:
+    api_url: str
+    api_key: str
+    timeout_seconds: int = 60  # 你也可以在反代侧控制超时
+
+
+class FirecrawlClient:
+    """
+    FirecrawlClient: 对 firecrawl python SDK 的封装 + 单例访问点。
+
+    - 提供 scrape / crawl 等常用方法，方便其他模块调用
+    - 线程安全单例（适合 Web 服务 / worker 多线程场景）
+    """
+
+    _instance: Optional["FirecrawlClient"] = None
+    _lock = threading.Lock()
+
+    def __init__(self, config: FirecrawlSettings):
+        self._settings: FirecrawlSettings = config
+        self._app: Firecrawl = self._create_app(config)
+
+    @staticmethod
+    def _create_app(config: FirecrawlSettings) -> Firecrawl:
+        try:
+            return Firecrawl(api_url=config.api_url, api_key=config.api_key)
+        except TypeError:
+            return Firecrawl(api_url=config.api_url, api_key=config.api_key)
+
+    @classmethod
+    def get_instance(cls) -> "FirecrawlClient":
+        """
+        线程安全的单例获取。
+        - 首次调用可传 settings
+        - 之后重复调用可不传
+        """
+        if cls._instance is not None:
+            return cls._instance
+
+        with cls._lock:
+            if cls._instance is not None:
+                return cls._instance
+
+            config = FirecrawlSettings(
+                api_url=FIRECRAWL_API_URL,
+                api_key=FIRECRAWL_API_KEY,
+                timeout_seconds=FIRECRAWL_TIMEOUT_SECONDS,
+            )
+
+            cls._instance = cls(config)
+            return cls._instance
+
+    @classmethod
+    def reset_instance(cls) -> None:
+        """测试用：重置单例。"""
+        with cls._lock:
+            cls._instance = None
+
+    def scrape_url(
+            self,
+            url: str,
+            formats: Optional[List[str]] = None,
+            only_main_content: bool = True,
+            timeout_seconds: Optional[int] = None,
+            extra_params: Optional[Dict[str, Any]] = None,
+    ) -> Dict[str, Any]:
+        """
+        单页抓取（最常用）
+        """
+        params: Dict[str, Any] = {
+            "formats": formats or ["markdown"],
+            "onlyMainContent": only_main_content,
+        }
+        if extra_params:
+            params.update(extra_params)
+
+        # if timeout_seconds is None:
+        #     timeout_seconds = self._settings.timeout_seconds
+
+        try:
+            return self._app.scrape(url, formats=formats, only_main_content=only_main_content).model_dump(
+                exclude_none=True)
+        except Exception as e:
+            raise RuntimeError(f"Firecrawl scrape_url failed: url={url}") from e

app/services/scrapers/firecrawl_client/scraper.py

@@ -0,0 +1,168 @@
+import hashlib
+from urllib.parse import urlparse
+
+from openai import AsyncOpenAI
+from openai.types.chat import ChatCompletionSystemMessageParam, ChatCompletionUserMessageParam
+
+from app.config import OPENAI_API_KEY
+from app.models.metadata_item import MediaFile, MessageType
+from app.services.scrapers.scraper import Scraper, DataProcessor
+from app.services.scrapers.firecrawl_client import FirecrawlItem
+from app.services.scrapers.firecrawl_client.client import FirecrawlClient
+from app.utils.parse import get_html_text_length, wrap_text_into_html
+from app.utils.logger import logger
+
+FIRECRAWL_TEXT_LIMIT = 800
+
+# System prompt for LLM to extract article content
+ARTICLE_EXTRACTION_PROMPT = """You are an expert content extractor. Your task is to extract the main article content from the provided HTML.
+
+Instructions:
+1. Identify and extract ONLY the main article/post content
+2. Remove navigation, headers, footers, sidebars, ads, comments, and other non-article elements
+3. Preserve the article's structure (headings, paragraphs, lists, etc.)
+4. Keep important formatting like bold, italic, links, and images
+5. Return clean HTML containing only the article content
+6. If you cannot identify the main content, return the original HTML unchanged
+
+Return ONLY the extracted HTML content, no explanations or markdown."""
+
+
+class FirecrawlDataProcessor(DataProcessor):
+    """
+    FirecrawlDataProcessor: Process URLs using Firecrawl to extract content.
+    """
+
+    def __init__(self, url: str):
+        self.url: str = url
+        self._data: dict = {}
+        self.url_parser = urlparse(url)
+        self.id = hashlib.md5(url.encode()).hexdigest()[:16]
+        self._client: FirecrawlClient = FirecrawlClient.get_instance()
+
+    async def get_item(self) -> dict:
+        await self.process_data()
+        firecrawl_item = FirecrawlItem.from_dict(self._data)
+        return firecrawl_item.to_dict()
+
+    async def process_data(self) -> None:
+        await self._get_page_content()
+
+    async def _get_page_content(self) -> None:
+        try:
+            result = self._client.scrape_url(
+                url=self.url,
+                formats=["markdown", "html"],
+                only_main_content=True,
+            )
+            await self._process_firecrawl_result(result)
+        except Exception as e:
+            logger.error(f"Failed to scrape URL with Firecrawl: {e}")
+            raise
+
+    @staticmethod
+    async def parsing_article_body_by_llm(html_content: str) -> str:
+        """
+        Use LLM to extract the main article content from HTML.
+
+        Args:
+            html_content: Raw HTML content from Firecrawl
+
+        Returns:
+            Cleaned HTML containing only the main article content
+        """
+        if not html_content:
+            return html_content
+
+        if not OPENAI_API_KEY:
+            logger.warning("OPENAI_API_KEY not configured, skipping LLM parsing")
+            return html_content
+
+        try:
+            client = AsyncOpenAI(api_key=OPENAI_API_KEY)
+
+            # Truncate content if too long to avoid token limits
+            max_content_length = 50000
+            truncated_content = html_content[:max_content_length] if len(html_content) > max_content_length else html_content
+
+            response = await client.chat.completions.create(
+                model="gpt-4o-mini",
+                messages=[
+                    ChatCompletionSystemMessageParam(role="system", content=ARTICLE_EXTRACTION_PROMPT),
+                    ChatCompletionUserMessageParam(role="user", content=f"Extract the main article content from this HTML:\n\n{truncated_content}")
+                ],
+                temperature=0.1,
+                max_tokens=16000,
+            )
+
+            extracted_content = response.choices[0].message.content
+
+            if extracted_content:
+                logger.info("Successfully extracted article content using LLM")
+                return extracted_content.strip()
+            else:
+                logger.warning("LLM returned empty content, using original HTML")
+                return html_content
+
+        except Exception as e:
+            logger.error(f"Failed to parse article body with LLM: {e}")
+            return html_content
+
+    async def _process_firecrawl_result(self, result: dict) -> None:
+        metadata = result.get("metadata", {})
+        markdown_content = result.get("markdown", "")
+        html_content = result.get("html", "")
+
+        # Extract metadata fields
+        title = metadata.get("title", "") or metadata.get("ogTitle", "") or self.url
+        author = metadata.get("author", "") or metadata.get("ogSiteName", "") or self.url_parser.netloc
+        description = metadata.get("description", "") or metadata.get("ogDescription", "")
+
+        item_data = {
+            "id": self.id,
+            "category": "other",
+            "url": self.url,
+            "title": title,
+            "author": author,
+            "author_url": f"{self.url_parser.scheme}://{self.url_parser.netloc}",
+        }
+
+        # Process text content - use description or first part of markdown
+        text = description if description else markdown_content[:500]
+        item_data["text"] = text
+
+        html_content = await self.parsing_article_body_by_llm(html_content)
+
+        # Process HTML content
+        if html_content:
+            content = wrap_text_into_html(html_content, is_html=True)
+        else:
+            content = wrap_text_into_html(markdown_content, is_html=False)
+        item_data["content"] = content
+        item_data["raw_content"] = markdown_content
+
+        # Process media files - extract og:image if available
+        media_files = []
+        og_image = metadata.get("ogImage")
+        if og_image:
+            media_files.append(MediaFile(url=og_image, media_type="image"))
+
+        item_data["media_files"] = [m.to_dict() for m in media_files]
+
+        # Determine message type based on text length
+        item_data["message_type"] = (
+            MessageType.LONG
+            if get_html_text_length(content) > FIRECRAWL_TEXT_LIMIT
+            else MessageType.SHORT
+        )
+
+        self._data = item_data
+
+
+class FirecrawlScraper(Scraper):
+    """
+    FirecrawlScraper: Scraper implementation using Firecrawl for generic URL scraping.
+    """
+
+    async def get_processor_by_url(self, url: str) -> DataProcessor:
+        return FirecrawlDataProcessor(url)

app/services/scrapers/scraper_manager.py

@@ -3,6 +3,7 @@
 from app.utils.logger import logger
 from app.services.scrapers.bluesky.scraper import BlueskyScraper
 from app.services.scrapers.weibo.scraper import WeiboScraper
+from app.services.scrapers.firecrawl_client.scraper import FirecrawlScraper
 from app.config import (
     BLUESKY_USERNAME, BLUESKY_PASSWORD
 )
@@ -12,9 +13,12 @@ class ScraperManager:
 
     bluesky_scraper: Optional[BlueskyScraper] = None
     weibo_scraper: Optional[WeiboScraper] = None
+    firecrawl_scraper: Optional[FirecrawlScraper] = None
 
     scrapers = {"bluesky": bluesky_scraper,
-                "weibo": bluesky_scraper}
+                "weibo": weibo_scraper,
+                "other": firecrawl_scraper,
+                "unknown": firecrawl_scraper}
 
     @classmethod
     async def init_scrapers(cls):
@@ -28,6 +32,8 @@ async def init_scraper(cls, category: str) -> None:
                 scraper = await cls.init_bluesky_scraper()
             elif category == "weibo" and not cls.weibo_scraper:
                 scraper = await cls.init_weibo_scraper()
+            elif category in ["other", "unknown"] and not cls.firecrawl_scraper:
+                scraper = await cls.init_firecrawl_scraper()
             if scraper:
                 cls.scrapers[category] = scraper
         else:
@@ -44,3 +50,9 @@ async def init_bluesky_scraper(cls) -> BlueskyScraper:
     async def init_weibo_scraper(cls) -> WeiboScraper:
         weibo_scraper = WeiboScraper()
         return weibo_scraper
+
+    @classmethod
+    async def init_firecrawl_scraper(cls) -> FirecrawlScraper:
+        firecrawl_scraper = FirecrawlScraper()
+        return firecrawl_scraper
+