Test LLM Providers #4
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test LLM Providers | |
| on: | |
| # Run on manual trigger only | |
| workflow_dispatch: | |
| permissions: | |
| contents: write | |
| jobs: | |
| test-providers: | |
| runs-on: ubuntu-latest | |
| env: | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| CODESTRAL_API_KEY: ${{ secrets.CODESTRAL_API_KEY }} | |
| DASHSCOPE_API_KEY: ${{ secrets.DASHSCOPE_API_KEY }} | |
| DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} | |
| GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} | |
| GOOGLE_FREE_API_KEY: ${{ secrets.GOOGLE_FREE_API_KEY }} | |
| GROK_API_KEY: ${{ secrets.GROK_API_KEY }} | |
| GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} | |
| MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }} | |
| ZAI_API_KEY: ${{ secrets.ZAI_API_KEY }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.x" | |
| - name: Install llms-py package | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install llms-py | |
| - name: Verify installation | |
| run: | | |
| llms | |
| - name: Test providers | |
| id: test-providers | |
| continue-on-error: true | |
| run: | | |
| # Create output directory | |
| mkdir -p test-results | |
| # Initialize results file | |
| RESULTS_TXT="test-results/provider-test-results.txt" | |
| echo "LLM Provider Test Results" > $RESULTS_TXT | |
| echo "=========================" >> $RESULTS_TXT | |
| echo "Date: $(date -u '+%Y-%m-%d %H:%M:%S UTC')" >> $RESULTS_TXT | |
| echo "llms-py version: $(pip show llms-py | grep Version | cut -d' ' -f2)" >> $RESULTS_TXT | |
| echo "" >> $RESULTS_TXT | |
| # List of providers to test | |
| PROVIDERS=( | |
| "openrouter_free" | |
| "groq" | |
| "codestral" | |
| "openrouter" | |
| "google" | |
| "anthropic" | |
| "openai" | |
| "grok" | |
| "qwen" | |
| "z.ai" | |
| "qwen" | |
| "mistral" | |
| ) | |
| # Test each provider | |
| for provider in "${PROVIDERS[@]}"; do | |
| echo "Testing provider: $provider" >> $RESULTS_TXT | |
| echo "-----------------------------------" >> $RESULTS_TXT | |
| # Run the check command and capture output | |
| if output=$(llms --check "$provider" 2>&1); then | |
| echo "Status: PASS" >> $RESULTS_TXT | |
| echo "$output" >> $RESULTS_TXT | |
| else | |
| echo "Status: FAIL" >> $RESULTS_TXT | |
| echo "$output" >> $RESULTS_TXT | |
| fi | |
| echo "" >> $RESULTS_TXT | |
| echo "" >> $RESULTS_TXT | |
| done | |
| echo "=========================" >> $RESULTS_TXT | |
| echo "Test run completed" >> $RESULTS_TXT | |
| # Display results in console | |
| echo "" | |
| echo "==========================================" | |
| echo "Test Results Summary" | |
| echo "==========================================" | |
| cat $RESULTS_TXT | |
| echo "==========================================" | |
| - name: Upload test results | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: provider-test-results | |
| path: test-results/provider-test-results.txt | |
| retention-days: 90 | |
| - name: Save results to repository | |
| if: always() | |
| run: | | |
| # Create docs/checks directory if it doesn't exist | |
| mkdir -p docs/checks | |
| # Get current date in YYYY-MM-DD format | |
| DATE=$(date -u '+%Y-%m-%d') | |
| # Copy results to both latest.txt and dated file | |
| cp test-results/provider-test-results.txt docs/checks/latest.txt | |
| cp test-results/provider-test-results.txt docs/checks/${DATE}.txt | |
| echo "Results saved to:" | |
| echo " - docs/checks/latest.txt" | |
| echo " - docs/checks/${DATE}.txt" | |
| - name: Commit and push results | |
| if: always() | |
| run: | | |
| git config --local user.email "github-actions[bot]@users.noreply.github.com" | |
| git config --local user.name "github-actions[bot]" | |
| git add docs/checks/ | |
| # Check if there are changes to commit | |
| if git diff --staged --quiet; then | |
| echo "No changes to commit" | |
| else | |
| DATE=$(date -u '+%Y-%m-%d') | |
| git commit -m "Update LLM provider test results - ${DATE}" | |
| git push | |
| echo "Results committed and pushed successfully" | |
| fi | |