diff --git a/.github/schemas/signals.schema.json b/.github/schemas/signals.schema.json index ded4367..1130cf2 100644 --- a/.github/schemas/signals.schema.json +++ b/.github/schemas/signals.schema.json @@ -1,13 +1,14 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://github.com/petry-projects/.github/blob/main/.github/schemas/signals.schema.json", - "$comment": "version: 1.0.0 — must match SCHEMA_VERSION in collect-signals.sh; enforced by bats", + "$comment": "version: 1.1.0 — must match SCHEMA_VERSION in collect-signals.sh; enforced by bats", "title": "Feature Ideation Signals", "description": "Canonical contract between collect-signals.sh and the BMAD Analyst (Mary) prompt. Any change to this schema is a breaking change to the workflow.", "type": "object", "required": [ "schema_version", "scan_date", + "last_successful_run", "repo", "open_issues", "closed_issues_30d", @@ -28,6 +29,11 @@ "type": "string", "format": "date-time" }, + "last_successful_run": { + "description": "ISO-8601 timestamp of the previous successful workflow run; used as a feed checkpoint by the analyst to skip already-reviewed content.", + "type": "string", + "format": "date-time" + }, "repo": { "type": "string", "pattern": "^[^/]+/[^/]+$" diff --git a/.github/scripts/feature-ideation/collect-signals.sh b/.github/scripts/feature-ideation/collect-signals.sh index fd3b1b3..0a11f77 100755 --- a/.github/scripts/feature-ideation/collect-signals.sh +++ b/.github/scripts/feature-ideation/collect-signals.sh @@ -31,7 +31,7 @@ set -euo pipefail # if the constants drift, AND the bats `signals-schema: SCHEMA_VERSION # constant matches schema file` test enforces this in CI. # Caught by CodeRabbit review on PR petry-projects/.github#85. -SCHEMA_VERSION="1.0.0" +SCHEMA_VERSION="1.1.0" SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=lib/gh-safe.sh @@ -71,6 +71,43 @@ main() { local scan_date scan_date=$(date_now_iso) + # --- Feed checkpoint: last successful run ---------------------------------- + # Used by the analyst to skip feed entries already reviewed. The current run + # is still in-progress, so --status=success --limit=1 reliably returns the + # previous successful run. Falls back to 30 days ago on first-ever run or + # after a long gap so the initial scan is bounded. + # WORKFLOW_FILE — caller-supplied env var for repos that name their stub + # something other than the conventional "feature-ideation.yml". Defaults to + # the conventional name; no change needed for repos that follow the standard. + printf '[collect-signals] resolving feed checkpoint (last successful run)\n' >&2 + local last_successful_run _run_stderr _run_err + _run_stderr=$(mktemp) + last_successful_run=$(gh run list \ + --repo "$REPO" \ + --workflow="${WORKFLOW_FILE:-feature-ideation.yml}" \ + --status=success \ + --limit=1 \ + --json createdAt \ + --jq '.[0].createdAt // empty' \ + 2>"$_run_stderr" || true) + _run_err=$(cat "$_run_stderr") + rm -f "$_run_stderr" + # Validate that the result looks like an ISO-8601 datetime. The real `gh` + # CLI applies the --jq filter and emits a bare timestamp; in test environments + # the gh stub returns raw fixture JSON (without applying --jq), so we guard + # against that here rather than requiring every test to stub this extra call. + if [ -z "$last_successful_run" ] || [ "$last_successful_run" = "null" ] || \ + ! printf '%s' "$last_successful_run" | grep -qE '^[0-9]{4}-[0-9]{2}-[0-9]{2}T'; then + if [ -n "$_run_err" ]; then + printf '[collect-signals] gh run list warning: %s\n' "$_run_err" >&2 + fi + last_successful_run="$(date_days_ago 30)T00:00:00Z" + printf '[collect-signals] no prior successful run found; using 30-day fallback: %s\n' \ + "$last_successful_run" >&2 + else + printf '[collect-signals] feed checkpoint: %s\n' "$last_successful_run" >&2 + fi + local truncation_warnings='[]' # --- Open issues ----------------------------------------------------------- @@ -201,6 +238,7 @@ GRAPHQL "$bug_reports" \ "$REPO" \ "$scan_date" \ + "$last_successful_run" \ "$SCHEMA_VERSION" \ "$truncation_warnings") @@ -217,6 +255,7 @@ GRAPHQL printf -- '- **Bug reports:** %s\n' "$(jq '.bug_reports.count' "$output_path")" printf -- '- **Merged PRs (30d):** %s\n' "$(jq '.merged_prs_30d.count' "$output_path")" printf -- '- **Existing Ideas discussions:** %s\n' "$(jq '.ideas_discussions.count' "$output_path")" + printf -- '- **Feed checkpoint (last successful run):** %s\n' "$(jq -r '.last_successful_run' "$output_path")" local warn_count warn_count=$(jq '.truncation_warnings | length' "$output_path") if [ "$warn_count" -gt 0 ]; then diff --git a/.github/scripts/feature-ideation/lib/compose-signals.sh b/.github/scripts/feature-ideation/lib/compose-signals.sh index 6a0f5bd..1afd8e7 100755 --- a/.github/scripts/feature-ideation/lib/compose-signals.sh +++ b/.github/scripts/feature-ideation/lib/compose-signals.sh @@ -18,16 +18,17 @@ # $7 bug_reports # $8 repo (string, e.g. "petry-projects/talkterm") # $9 scan_date (ISO-8601 string) -# $10 schema_version (string) -# $11 truncation_warnings (JSON array, may be []) +# $10 last_successful_run (ISO-8601 string; feed checkpoint) +# $11 schema_version (string) +# $12 truncation_warnings (JSON array, may be []) # # Output: signals.json document on stdout. set -euo pipefail compose_signals() { - if [ "$#" -ne 11 ]; then - printf '[compose-signals] expected 11 args, got %d\n' "$#" >&2 + if [ "$#" -ne 12 ]; then + printf '[compose-signals] expected 12 args, got %d\n' "$#" >&2 return 64 # EX_USAGE fi @@ -40,8 +41,9 @@ compose_signals() { local bug_reports="$7" local repo="$8" local scan_date="$9" - local schema_version="${10}" - local truncation_warnings="${11}" + local last_successful_run="${10}" + local schema_version="${11}" + local truncation_warnings="${12}" # Validate every JSON input before composition. Better to fail loudly here # than to let `jq --argjson` produce a cryptic parse error. @@ -57,6 +59,7 @@ compose_signals() { jq -n \ --arg scan_date "$scan_date" \ + --arg last_successful_run "$last_successful_run" \ --arg repo "$repo" \ --arg schema_version "$schema_version" \ --argjson open_issues "$open_issues" \ @@ -70,6 +73,7 @@ compose_signals() { '{ schema_version: $schema_version, scan_date: $scan_date, + last_successful_run: $last_successful_run, repo: $repo, open_issues: { count: ($open_issues | length), items: $open_issues }, closed_issues_30d: { count: ($closed_issues | length), items: $closed_issues }, diff --git a/.github/workflows/feature-ideation-reusable.yml b/.github/workflows/feature-ideation-reusable.yml index c008175..c158006 100644 --- a/.github/workflows/feature-ideation-reusable.yml +++ b/.github/workflows/feature-ideation-reusable.yml @@ -20,7 +20,9 @@ # Why a reusable workflow: # - The 5-phase ideation pipeline (Market Research → Brainstorming → Party # Mode → Adversarial → Publish) is universal. Tuning the pattern in one -# place propagates to every BMAD-enabled repo on next run. +# place propagates to every BMAD-enabled repo after the v1 tag is bumped +# and their next scheduled run executes. Repos tracking @main pick it up +# immediately on their next run. # - The two critical gotchas (github_token override, ANTHROPIC_MODEL env var) # are baked in here so callers cannot accidentally regress them. # @@ -76,6 +78,16 @@ on: required: false default: 'v1' type: string + sources_file: + description: | + Path (relative to repo root) of the repo-local reputable source list. + Defaults to .github/feature-ideation-sources.md. Copy + standards/feature-ideation-sources.md from petry-projects/.github as + a starting point and customise it for your project. If the file is + absent Mary falls back to open web search automatically. + required: false + default: '.github/feature-ideation-sources.md' + type: string secrets: CLAUDE_CODE_OAUTH_TOKEN: description: 'Claude Code OAuth token (org-level secret)' @@ -94,6 +106,7 @@ jobs: issues: read pull-requests: read discussions: read + actions: read env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} steps: @@ -199,6 +212,7 @@ jobs: MATCH_PLAN_PATH: ${{ runner.temp }}/match-plan.json TOOLING_DIR: ${{ github.workspace }}/.feature-ideation-tooling/.github/scripts/feature-ideation FOCUS_AREA: ${{ inputs.focus_area || '' }} + SOURCES_FILE_PATH: ${{ inputs.sources_file }} RESEARCH_DEPTH: ${{ inputs.research_depth }} uses: anthropics/claude-code-action@905d4eb99ab3d43143d74fb0dcae537f29ac330a # v1.0.97 with: @@ -295,8 +309,58 @@ jobs: **Adopt the "Market Research" mindset.** Your sole job in this phase is to gather hard evidence — not to generate ideas yet. Be a detective, not a brainstormer. - Using the **Project Context** above as your starting point, research the - competitive landscape and emerging trends. Cover at minimum: + ### Start from the repo-local Reputable Source List + + Before running open-ended web searches, **check for a repo-local source list**. + If `$SOURCES_FILE_PATH` is non-empty, read it: + + ``` + Read: $SOURCES_FILE_PATH + ``` + + If the path is empty or the file does not exist, log a one-line warning + in the step summary under "Source list status" and proceed with open web search. + + That file (if present) lists vendor blogs, RSS feeds, podcasts, and YouTube + channels this repo's maintainers trust as primary signal sources, organised + by category. Treat it as your **starting set**: + + - Pick the categories most relevant to the **Project Context** above. A + CLI dev tool warrants different sources than a security scanner or a + data pipeline. + - For each relevant source, do a targeted `WebFetch` or `WebSearch` + scoped to that domain to find what's new. Prefer fetching the listed + RSS/Atom feed URL when one is given — it gives you structured "what + changed" data without scraping HTML. + - The list is a **floor, not a ceiling.** If a thread takes you to a + source not in the list and that source is reputable, follow it. After + the run, if a high-value source kept coming up but wasn't listed, + **mention it in the step summary** under a "Suggested additions to + source list" heading so a human can PR it in. + - If the file is missing or empty, log a warning to the step summary + and proceed with open web search — do not fail the run. + + ### Use the feed checkpoint to avoid re-reviewing old content + + `$SIGNALS_PATH` contains a `last_successful_run` ISO-8601 timestamp — the + date/time the previous successful run of this workflow completed. + + **When fetching any RSS/Atom feed or scraping a changelog/blog:** + - Only consider entries whose `pubDate` / `updated` / `published` field + is **after** `last_successful_run`. + - If a feed does not expose dates, or if all entries predate the checkpoint, + note that in your research log and move on — do not re-summarise content + you have no evidence is new. + - If `last_successful_run` is more than 60 days in the past (e.g. after a + long workflow outage), ignore the checkpoint and process the full feed to + avoid missing major developments. + + This keeps each weekly run focused on genuinely new signal rather than + re-processing the same content at Opus 4.6 cost. + + Using the **Project Context** above as your starting point and the + source list as your seed, research the competitive landscape and + emerging trends. Cover at minimum: ### Competitive Landscape - What are competitors in this space shipping recently? diff --git a/.github/workflows/feature-ideation.yml b/.github/workflows/feature-ideation.yml new file mode 100644 index 0000000..afea467 --- /dev/null +++ b/.github/workflows/feature-ideation.yml @@ -0,0 +1,88 @@ +# ───────────────────────────────────────────────────────────────────────────── +# SOURCE OF TRUTH: petry-projects/.github/standards/workflows/feature-ideation.yml +# Standard: petry-projects/.github/standards/ci-standards.md#8 +# Reusable: petry-projects/.github/.github/workflows/feature-ideation-reusable.yml +# +# AGENTS — READ BEFORE EDITING: +# • This file is a THIN CALLER STUB. The 5-phase ideation pipeline, the +# Opus 4.6 model selection, the github_token override, and the +# ANTHROPIC_MODEL env var all live in the reusable workflow above. +# • You MAY change: the `project_context` value and optionally the cron +# schedule. +# • You MUST NOT change: trigger event shape, the `uses:` line, the +# job-level `permissions:` block, or the `secrets:` block. +# • If you need different behaviour, open a PR against the reusable in +# this repo. The change will propagate after the v1 tag is bumped. +# ───────────────────────────────────────────────────────────────────────────── +name: Feature Research & Ideation (BMAD Analyst) + +on: + schedule: + - cron: '0 7 * * 5' # Friday 07:00 UTC (3 AM EDT / 2 AM EST) + workflow_dispatch: + inputs: + focus_area: + description: 'Optional focus area (e.g., "CI patterns", "agent security")' + required: false + type: string + research_depth: + description: 'Research depth' + required: false + default: 'standard' + type: choice + options: + - quick + - standard + - deep + dry_run: + description: 'Log Discussion mutations to artifact instead of writing live' + required: false + default: false + type: boolean + +permissions: {} + +concurrency: + group: feature-ideation + cancel-in-progress: false + +jobs: + ideate: + # Permissions cascade from the calling job to the reusable workflow. + # The reusable workflow's two jobs (gather-signals + analyze) need: + # - contents: read (checkout, file reads) + # - issues: read (signal collection) + # - pull-requests: read (signal collection) + # - discussions: write (CRITICAL — create/update Discussion threads) + # - actions: read (feed checkpoint via gh run list) + # - id-token: write (claude-code-action OIDC for GitHub App token) + permissions: + contents: read + issues: read + pull-requests: read + discussions: write + actions: read + id-token: write + uses: petry-projects/.github/.github/workflows/feature-ideation-reusable.yml@208ec2d69b75227d375edf8745d84fbac05a76b2 # v1 — bump SHA after tag update + with: + project_context: | + petry-projects/.github is the org-level standards and tooling repository + for the petry-projects GitHub organisation. It owns the canonical CI/CD + standards (ci-standards.md), all reusable GitHub Actions workflows + (claude-code, feature-ideation, agent-shield, dependabot), the BMAD + Method agentic development framework, and org-wide security and + compliance tooling. Downstream consumers are internal product repos + (Broodly, TalkTerm, markets). Key trends to track: GitHub Actions + platform improvements (larger runners, step summaries, OIDC advances), + agentic CI patterns (LLM-driven PR review, automated ideation, + agent-shield threat models), AI-assisted code review tooling evolution + (CodeRabbit, Copilot Workspace), security hardening for AI-agent + workflows (prompt injection, supply-chain pinning, least-privilege + scopes), and developer-experience standards for LLM-heavy codebases. + No public users — this is internal DevX infrastructure. + sources_file: 'standards/feature-ideation-sources.md' + focus_area: ${{ inputs.focus_area || '' }} + research_depth: ${{ inputs.research_depth || 'standard' }} + dry_run: ${{ inputs.dry_run || false }} + secrets: + CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} diff --git a/standards/ci-standards.md b/standards/ci-standards.md index 6f2d7b5..be3fe74 100644 --- a/standards/ci-standards.md +++ b/standards/ci-standards.md @@ -22,7 +22,7 @@ where to send a fix when behavior needs to change. | Tier | Examples | What lives in `standards/workflows/` | Where logic lives | Edits allowed in adopting repo | |---|---|---|---|---| -| **1. Stub** | `claude.yml`, `dependency-audit.yml`, `dependabot-automerge.yml`, `dependabot-rebase.yml`, `agent-shield.yml`, `feature-ideation.yml` | A thin caller stub that delegates via `uses: petry-projects/.github/.github/workflows/-reusable.yml@v1` | The matching `*-reusable.yml` in this repo (single source of truth) | **None** in normal use. May tune `with:` inputs where the reusable exposes them (e.g. `agent-shield` accepts `min-severity`, `required-files`; `feature-ideation` requires `project_context`). To change behavior, open a PR against the reusable in this repo — the change propagates everywhere on next run. | +| **1. Stub** | `claude.yml`, `dependency-audit.yml`, `dependabot-automerge.yml`, `dependabot-rebase.yml`, `agent-shield.yml`, `feature-ideation.yml` | A thin caller stub that delegates via `uses: petry-projects/.github/.github/workflows/-reusable.yml@v1` | The matching `*-reusable.yml` in this repo (single source of truth) | **None** in normal use. May tune `with:` inputs where the reusable exposes them (e.g. `agent-shield` accepts `min-severity`, `required-files`; `feature-ideation` requires `project_context`). To change behavior, open a PR against the reusable in this repo — repos on `@v1` pick it up after the `v1` tag is bumped; repos on `@main` pick it up on their next run. | | **2. Per-repo template** | `ci.yml`, `sonarcloud.yml` | _(no template — see the patterns documented below)_ | In each repo, because the workflow is tech-stack-specific (language matrix, build tool, test framework) | **Limited.** Each adopting repo carries its own copy. Stay within the patterns in this document; do not change action SHAs, permission scopes, trigger events, or job names without raising a standards PR first. | | **GitHub-managed** | CodeQL default setup | _(no workflow file — managed via repo Settings → Code security)_ | GitHub | None. Configured via `apply-repo-settings.sh`; per-repo `codeql.yml` files are treated as drift by the compliance audit. See [§2 CodeQL Analysis](#2-codeql-analysis-github-managed-default-setup). | | **3. Free per-repo** | `release.yml`, project-specific automation | _(out of scope for this standard)_ | Per-repo | Free, but must still comply with the [Action Pinning Policy](#action-pinning-policy) and the [Required Workflows](#required-workflows) constraints. | @@ -475,8 +475,21 @@ split into two parts: Defines the schedule, the `workflow_dispatch` inputs, and calls the reusable workflow with a single required parameter: `project_context`. +3. **Reputable Source List** (repo-local, per-repo): + Each adopting repo maintains its own copy at `.github/feature-ideation-sources.md` + (or the path passed via the `sources_file` workflow input). + Use [`standards/feature-ideation-sources.md`](feature-ideation-sources.md) + as a starter template, then customise it for your project. The Phase 2 prompt + instructs Mary to read that file as her **starting set** for market research — + vendor blogs, RSS feeds, podcasts, and YouTube channels organised by category. + If the file is absent Mary falls back to open web search automatically. + Each repo owns its own copy; add or remove entries via PR in that repo. + When we tune the prompt, the model, or the gotchas, we change one file in -this repo and every adopter picks up the change on their next scheduled run. +this repo. Repos tracking `@main` pick up the change on their next scheduled +run; repos pinned to `@v1` pick it up only after the `v1` tag is updated and +then on their next scheduled run. The source list is repo-local and propagates +only within the repo that owns it. #### Adopting in a new repo @@ -485,11 +498,15 @@ this repo and every adopter picks up the change on their next scheduled run. 2. Replace the `project_context` value with a 3-5 sentence description of what the project is, who it serves, and the competitive landscape Mary should research. This is the **only** required edit. -3. (Optional) Adjust the cron schedule, focus area choices, or pin to a +3. (Optional) Copy [`standards/feature-ideation-sources.md`](feature-ideation-sources.md) + to `.github/feature-ideation-sources.md` in the target repo and customise + it for your project. Mary reads YOUR copy — not the central template — so + each repo controls its own source list. +4. (Optional) Adjust the cron schedule, focus area choices, or pin to a tag instead of `@main` if you want change isolation. -4. Ensure GitHub Discussions is enabled with an "Ideas" category — see +5. Ensure GitHub Discussions is enabled with an "Ideas" category — see [Discussions Configuration](github-settings.md#discussions-configuration). -5. Confirm the org-level secret `CLAUDE_CODE_OAUTH_TOKEN` is accessible. +6. Confirm the org-level secret `CLAUDE_CODE_OAUTH_TOKEN` is accessible. #### Critical gotchas (baked into the reusable workflow) diff --git a/standards/feature-ideation-sources.md b/standards/feature-ideation-sources.md new file mode 100644 index 0000000..1165e2a --- /dev/null +++ b/standards/feature-ideation-sources.md @@ -0,0 +1,174 @@ +# Feature Ideation — Reputable Source List + +> **Purpose:** A starter list of reputable web pages, RSS feeds, podcasts, +> and YouTube channels that the BMAD Analyst (Mary) consults during +> **Phase 2: Market Research** of the +> [Feature Ideation workflow](../.github/workflows/feature-ideation-reusable.yml). +> +> **How it is used:** This file is a **template**. Each adopting repo copies +> it to `.github/feature-ideation-sources.md` (or the path configured via +> the `sources_file` workflow input) and customises it for their project. +> The reusable workflow reads the repo-local copy — it does **not** read +> this file directly. Mary treats the local list as her starting set for +> web research, supplementing it with targeted searches as needed. +> +> **Curation rules:** +> +> - Only sources with a track record of accurate, technically-grounded +> content. No SEO farms, no anonymous aggregators with no editorial +> standards. +> - Prefer **primary sources** (vendor changelogs, research labs, official +> blogs) over secondary commentary. +> - RSS / Atom feed URLs are listed where available — they let Mary fetch +> structured "what is new since last scan" data instead of scraping HTML. +> - YouTube and podcast feeds are included because release announcements, +> conference talks, and engineering deep-dives often appear there before +> (or instead of) blog posts. +> - Every entry has a one-line note explaining **why it is on this list** — +> if you cannot justify it in one line, it should not be here. +> +> **Maintenance:** Each repo owns its own copy — add or remove entries via +> PR in that repo. To update the shared starter template, open a PR here; +> existing repos with their own copy will not be affected automatically. + +--- + +## 1. AI / ML — Vendor & Lab Primary Sources + +Release notes, model launches, capability changes. These are usually the +**first** place a new feature surfaces, weeks before commentary catches up. + +| Source | URL | Feed | Why it's here | +|--------|-----|------|---------------| +| Anthropic News | | | Model releases, safety research, API changes | +| OpenAI Blog | | | Model releases, API updates, research | +| Google DeepMind Blog | | — | Research breakthroughs, Gemini updates | +| Google AI Blog | | | Applied AI research and product updates | +| Meta AI Blog | | — | LLaMA releases, research | +| Mistral Blog | | — | Open-weight model releases | +| HuggingFace Blog | | | Model releases, dataset news, community trends | +| Cohere Blog | | — | Enterprise LLM API updates | +| xAI Blog | | — | Grok model updates | + +## 2. AI / ML — Research & Trends + +Pre-prints, paper trackers, and analyst commentary. Useful for spotting +emerging capabilities **before** they hit vendor APIs. + +| Source | URL | Feed | Why it's here | +|--------|-----|------|---------------| +| arXiv cs.AI | | | Daily AI pre-prints | +| arXiv cs.CL | | | NLP / language model pre-prints | +| arXiv cs.LG | | | Machine learning pre-prints | +| Papers with Code — Trending | | — | Papers ranked by community attention with reference implementations | +| Import AI (Jack Clark) | | | Weekly research roundup with policy + capability framing | +| The Gradient | | | Long-form ML research essays | + +## 3. Developer Tooling, DevEx & Platform Changelogs + +Platform features that unlock new product capabilities. GitHub's changelog +in particular is the **single most important feed** for any project hosted +on GitHub. + +| Source | URL | Feed | Why it's here | +|--------|-----|------|---------------| +| GitHub Changelog | | | GitHub product feature releases | +| GitHub Engineering | | | Deep-dives on GitHub's own engineering decisions | +| GitLab Blog | | | DevOps platform changes | +| Vercel Blog | | — | Frontend/edge platform updates | +| Cloudflare Blog | | | Edge, Workers, security product updates | +| AWS What's New | | | AWS service launches and updates | +| GCP Blog | | | GCP release notes | +| Stack Overflow Blog | | | Developer survey data, industry trends | +| The Pragmatic Engineer | | — | Engineering leadership and tooling trends | + +## 4. Security & Compliance + +Vulnerabilities, supply chain threats, and compliance changes that may +surface as feature requirements. + +| Source | URL | Feed | Why it's here | +|--------|-----|------|---------------| +| GitHub Security Blog | | | GitHub's own security advisories and features | +| GitHub Advisory Database | | — | Known vulnerabilities in open-source packages | +| CISA Known Exploited Vulnerabilities | | | US government KEV feed | +| OpenSSF Blog | | | Supply chain security standards | +| Snyk Blog | | | Vulnerability research and DevSecOps trends | +| Krebs on Security | | | Investigative security journalism | + +## 5. Software Engineering Practice & Industry Analysis + +Long-form analysis of engineering decisions, industry shifts, and +developer trends. + +| Source | URL | Feed | Why it's here | +|--------|-----|------|---------------| +| Hacker News — Top | | | Community signal on what engineers care about | +| Lobsters | | | Curated engineering discussion | +| Martin Fowler | | | Software design patterns and architecture | +| Latent Space (substack) | | | AI engineering deep-dives | +| Simon Willison's Weblog | | | Practical LLM applications and web tech | +| Stratechery | | — | Tech strategy and business model analysis | +| Increment | | | Long-form engineering practice essays | + +## 6. Newsletters + +Curated weekly signal with low noise. Prefer fetching the RSS feed if +listed over the web version. + +| Source | URL | Feed | Why it's here | +|--------|-----|------|---------------| +| TLDR | | | High-signal daily tech summary | +| Ben's Bites | | | Daily AI product and research digest | +| The Rundown AI | | — | AI tools and model updates for practitioners | +| Bytes (JS) | | — | JavaScript/TypeScript ecosystem news | +| Python Weekly | | — | Python ecosystem packages and tutorials | +| DevOps Weekly | | — | DevOps tooling and practices | + +## 7. Podcasts + +Engineering and product insights that often precede written coverage. +Subscribe to the RSS feed and look for recent episode titles/descriptions. + +| Source | URL | Feed | Why it's here | +|--------|-----|------|---------------| +| Latent Space Podcast | | | AI engineering interviews with practitioners | +| The Changelog | | | Open-source and developer tooling | +| Practical AI | | | Applied ML and AI products | +| a16z Podcast | | | Tech and startup strategy | +| The Cognitive Revolution | | — | AI capability and safety interviews | +| Lex Fridman Podcast | | | Long-form researcher/founder interviews | +| Software Engineering Daily | | | Engineering deep-dives | + +## 8. YouTube Channels + +Conference talks and product launches often land on YouTube before +blog posts. Fetch the channel's RSS feed to see recent video titles. + +| Source | Channel URL | RSS Feed | Why it's here | +|--------|-------------|----------|---------------| +| Fireship | | | Quick-hit tech trends and new tool releases | +| ThePrimeagen | | | Developer tooling opinions and Rust/Go/TS trends | +| Two Minute Papers | | | Accessible ML research summaries | +| Yannic Kilcher | | | Deep ML paper walkthroughs | +| AI Explained | | | LLM capability updates | +| Matthew Berman | | | AI tool demos and model comparisons | +| GitHub on YouTube | | | GitHub product demos and Universe talks | +| AWS Events | | | re:Invent, re:Inforce session recordings | + +## 9. Conferences + +Major annual events where product announcements and research previews +cluster. Monitor the conference site and YouTube channel in the weeks +before and after each event. + +| Conference | Typical date | URL | Why it's here | +|------------|-------------|-----|---------------| +| GitHub Universe | Oct | | GitHub roadmap and ecosystem announcements | +| KubeCon / CloudNativeCon | Mar + Nov | | Cloud-native platform shifts | +| AWS re:Invent | Nov–Dec | | AWS service launches | +| Google Cloud Next | Apr | | GCP and Workspace announcements | +| NeurIPS | Dec | | Top-tier ML research | +| ICML | Jul | | Machine learning research trends | +| ICLR | May | | Deep learning and representation learning | +| Strange Loop (archive) | — | | Engineering practice talks (archived; still valuable) | diff --git a/standards/workflows/feature-ideation.yml b/standards/workflows/feature-ideation.yml index cb03d75..b8a4c05 100644 --- a/standards/workflows/feature-ideation.yml +++ b/standards/workflows/feature-ideation.yml @@ -28,9 +28,15 @@ # 2. Replace the `project_context` value with a 3-5 sentence description # of your project, its target users, and the competitive landscape Mary # should research. This is the only required customisation. -# 3. (Optional) Adjust the schedule cron if Friday morning UTC doesn't suit. -# 4. Ensure GitHub Discussions is enabled with an "Ideas" category. -# 5. Confirm the org-level secret CLAUDE_CODE_OAUTH_TOKEN is accessible. +# 3. (Optional) Copy standards/feature-ideation-sources.md from +# petry-projects/.github to .github/feature-ideation-sources.md in your +# repo and trim/extend it for your project. Mary uses YOUR copy — not the +# central template — so each repo controls its own source list. +# Pass `sources_file: path/to/your-list.md` to the reusable workflow if +# you prefer a different location. +# 4. (Optional) Adjust the schedule cron if Friday morning UTC doesn't suit. +# 5. Ensure GitHub Discussions is enabled with an "Ideas" category. +# 6. Confirm the org-level secret CLAUDE_CODE_OAUTH_TOKEN is accessible. # # Standard: https://github.com/petry-projects/.github/blob/main/standards/ci-standards.md#8-feature-ideation-feature-ideationyml--bmad-method-repos name: Feature Research & Ideation (BMAD Analyst) @@ -69,18 +75,20 @@ jobs: ideate: # Permissions cascade from the calling job to the reusable workflow. # The reusable workflow's two jobs (gather-signals + analyze) need: - # - contents: read (checkout, file reads) - # - issues: read (signal collection) - # - pull-requests: read (signal collection) - # - discussions: write (CRITICAL — create/update Discussion threads) - # - id-token: write (claude-code-action OIDC for GitHub App token) + # - contents: read (checkout, file reads) + # - issues: read (signal collection) + # - pull-requests: read (signal collection) + # - discussions: write (CRITICAL — create/update Discussion threads) + # - id-token: write (claude-code-action OIDC for GitHub App token) + # - actions: read (feed checkpoint — last successful run query) permissions: contents: read issues: read pull-requests: read discussions: write id-token: write - uses: petry-projects/.github/.github/workflows/feature-ideation-reusable.yml@v1 + actions: read + uses: petry-projects/.github/.github/workflows/feature-ideation-reusable.yml@ae9709f4466dec60a5733c9e7487f69dcd004e05 # v1 with: # === CUSTOMISE THIS PER REPO — the only required edit === # Replace this paragraph with a 3-5 sentence description of your project, @@ -90,6 +98,12 @@ jobs: TODO: Replace this with a description of the project and its market. Example: "ProjectX is a [type of product] for [target user]. Competitors include A, B, C. Key emerging trends in this space: X, Y, Z." + # === OPTIONAL: repo-local reputable source list === + # Copy standards/feature-ideation-sources.md from petry-projects/.github + # to .github/feature-ideation-sources.md and customise it. The reusable + # workflow defaults to that path, so you only need to uncomment and change + # sources_file below if you store the list somewhere else. + # sources_file: 'docs/feature-ideation-sources.md' focus_area: ${{ inputs.focus_area || '' }} research_depth: ${{ inputs.research_depth || 'standard' }} dry_run: ${{ inputs.dry_run || false }} diff --git a/test/workflows/feature-ideation/collect-signals.bats b/test/workflows/feature-ideation/collect-signals.bats index 94f9a12..cf306ff 100644 --- a/test/workflows/feature-ideation/collect-signals.bats +++ b/test/workflows/feature-ideation/collect-signals.bats @@ -21,15 +21,17 @@ teardown() { # Build a multi-call gh script for the standard happy path. # Order MUST match collect-signals.sh: -# 1. gh issue list --state open -# 2. gh issue list --state closed -# 3. gh api graphql (categories) -# 4. gh api graphql (discussions) -# 5. gh release list -# 6. gh pr list --state merged +# 1. gh run list (feed checkpoint — last successful run) +# 2. gh issue list --state open +# 3. gh issue list --state closed +# 4. gh api graphql (categories) +# 5. gh api graphql (discussions) +# 6. gh release list +# 7. gh pr list --state merged build_happy_script() { local script="${TT_TMP}/gh-script.tsv" : >"$script" + printf '0\t%s\t-\n' "${TT_FIXTURES_DIR}/gh-responses/run-list-last-success.txt" >>"$script" printf '0\t%s\t-\n' "${TT_FIXTURES_DIR}/gh-responses/issue-list-open.json" >>"$script" printf '0\t%s\t-\n' "${TT_FIXTURES_DIR}/gh-responses/issue-list-closed.json" >>"$script" printf '0\t%s\t-\n' "${TT_FIXTURES_DIR}/gh-responses/graphql-categories.json" >>"$script" @@ -115,7 +117,9 @@ build_happy_script() { script="${TT_TMP}/gh-script.tsv" err_file="${TT_TMP}/auth-err.txt" printf 'HTTP 401: Bad credentials\n' >"$err_file" - printf '4\t-\t%s\n' "$err_file" >"$script" + # run list (feed checkpoint — silenced with || true, so failure falls back) + printf '0\t%s\t-\n' "${TT_FIXTURES_DIR}/gh-responses/run-list-last-success.txt" >"$script" + printf '4\t-\t%s\n' "$err_file" >>"$script" export GH_STUB_SCRIPT="$script" rm -f "${TT_TMP}/.gh-stub-counter" @@ -127,6 +131,7 @@ build_happy_script() { @test "collect-signals: FAILS LOUD on GraphQL errors envelope (categories)" { script="${TT_TMP}/gh-script.tsv" : >"$script" + printf '0\t%s\t-\n' "${TT_FIXTURES_DIR}/gh-responses/run-list-last-success.txt" >>"$script" printf '0\t%s\t-\n' "${TT_FIXTURES_DIR}/gh-responses/issue-list-open.json" >>"$script" printf '0\t%s\t-\n' "${TT_FIXTURES_DIR}/gh-responses/issue-list-closed.json" >>"$script" printf '0\t%s\t-\n' "${TT_FIXTURES_DIR}/gh-responses/graphql-errors-envelope.json" >>"$script" @@ -183,6 +188,7 @@ JSON script="${TT_TMP}/gh-script.tsv" : >"$script" + printf '0\t%s\t-\n' "${TT_FIXTURES_DIR}/gh-responses/run-list-last-success.txt" >>"$script" # feed checkpoint printf '0\t%s\t-\n' "$bot_file" >>"$script" # open issues — all bots printf '0\t%s\t-\n' "$empty_file" >>"$script" # closed issues printf '0\t%s\t-\n' "${TT_FIXTURES_DIR}/gh-responses/graphql-no-ideas-category.json" >>"$script" @@ -202,6 +208,7 @@ JSON @test "collect-signals: emits truncation warning when discussions hasNextPage=true" { script="${TT_TMP}/gh-script.tsv" : >"$script" + printf '0\t%s\t-\n' "${TT_FIXTURES_DIR}/gh-responses/run-list-last-success.txt" >>"$script" printf '0\t%s\t-\n' "${TT_FIXTURES_DIR}/gh-responses/issue-list-open.json" >>"$script" printf '0\t%s\t-\n' "${TT_FIXTURES_DIR}/gh-responses/issue-list-closed.json" >>"$script" printf '0\t%s\t-\n' "${TT_FIXTURES_DIR}/gh-responses/graphql-categories.json" >>"$script" @@ -225,6 +232,7 @@ JSON @test "collect-signals: skips discussions when Ideas category absent" { script="${TT_TMP}/gh-script.tsv" : >"$script" + printf '0\t%s\t-\n' "${TT_FIXTURES_DIR}/gh-responses/run-list-last-success.txt" >>"$script" printf '0\t%s\t-\n' "${TT_FIXTURES_DIR}/gh-responses/issue-list-open.json" >>"$script" printf '0\t%s\t-\n' "${TT_FIXTURES_DIR}/gh-responses/issue-list-closed.json" >>"$script" printf '0\t%s\t-\n' "${TT_FIXTURES_DIR}/gh-responses/graphql-no-ideas-category.json" >>"$script" diff --git a/test/workflows/feature-ideation/compose-signals.bats b/test/workflows/feature-ideation/compose-signals.bats index 4dced2c..35359e9 100644 --- a/test/workflows/feature-ideation/compose-signals.bats +++ b/test/workflows/feature-ideation/compose-signals.bats @@ -18,6 +18,7 @@ compose_empty() { '[]' '[]' '[]' '[]' '[]' '[]' '[]' \ 'foo/bar' \ '2026-04-07T00:00:00Z' \ + '2026-03-07T00:00:00Z' \ '1.0.0' \ '[]' } @@ -34,14 +35,14 @@ compose_empty() { @test "compose: rejects empty string for any JSON arg" { run compose_signals \ '' '[]' '[]' '[]' '[]' '[]' '[]' \ - 'foo/bar' '2026-04-07T00:00:00Z' '1.0.0' '[]' + 'foo/bar' '2026-04-07T00:00:00Z' '2026-03-07T00:00:00Z' '1.0.0' '[]' [ "$status" -ne 0 ] } @test "compose: rejects non-JSON for any JSON arg" { run compose_signals \ 'not json' '[]' '[]' '[]' '[]' '[]' '[]' \ - 'foo/bar' '2026-04-07T00:00:00Z' '1.0.0' '[]' + 'foo/bar' '2026-04-07T00:00:00Z' '2026-03-07T00:00:00Z' '1.0.0' '[]' [ "$status" -ne 0 ] } @@ -52,9 +53,9 @@ compose_empty() { @test "compose: produces all required top-level fields with empty inputs" { run compose_empty [ "$status" -eq 0 ] - for field in schema_version scan_date repo open_issues closed_issues_30d \ - ideas_discussions releases merged_prs_30d feature_requests \ - bug_reports truncation_warnings; do + for field in schema_version scan_date last_successful_run repo open_issues \ + closed_issues_30d ideas_discussions releases merged_prs_30d \ + feature_requests bug_reports truncation_warnings; do printf '%s' "$output" | jq -e "has(\"$field\")" >/dev/null done } @@ -63,7 +64,7 @@ compose_empty() { open='[{"number":1,"title":"a","labels":[]},{"number":2,"title":"b","labels":[]}]' run compose_signals \ "$open" '[]' '[]' '[]' '[]' '[]' '[]' \ - 'foo/bar' '2026-04-07T00:00:00Z' '1.0.0' '[]' + 'foo/bar' '2026-04-07T00:00:00Z' '2026-03-07T00:00:00Z' '1.0.0' '[]' [ "$status" -eq 0 ] count=$(printf '%s' "$output" | jq '.open_issues.count') items_len=$(printf '%s' "$output" | jq '.open_issues.items | length') @@ -74,7 +75,7 @@ compose_empty() { @test "compose: schema_version is preserved verbatim" { run compose_signals \ '[]' '[]' '[]' '[]' '[]' '[]' '[]' \ - 'foo/bar' '2026-04-07T00:00:00Z' '2.5.1' '[]' + 'foo/bar' '2026-04-07T00:00:00Z' '2026-03-07T00:00:00Z' '2.5.1' '[]' [ "$status" -eq 0 ] v=$(printf '%s' "$output" | jq -r '.schema_version') [ "$v" = "2.5.1" ] @@ -84,7 +85,7 @@ compose_empty() { warnings='[{"source":"open_issues","limit":50,"message":"truncated"}]' run compose_signals \ '[]' '[]' '[]' '[]' '[]' '[]' '[]' \ - 'foo/bar' '2026-04-07T00:00:00Z' '1.0.0' "$warnings" + 'foo/bar' '2026-04-07T00:00:00Z' '2026-03-07T00:00:00Z' '1.0.0' "$warnings" [ "$status" -eq 0 ] src=$(printf '%s' "$output" | jq -r '.truncation_warnings[0].source') [ "$src" = "open_issues" ] @@ -93,7 +94,7 @@ compose_empty() { @test "compose: scan_date and repo round-trip exactly" { run compose_signals \ '[]' '[]' '[]' '[]' '[]' '[]' '[]' \ - 'octocat/hello-world' '2030-01-15T12:34:56Z' '1.0.0' '[]' + 'octocat/hello-world' '2030-01-15T12:34:56Z' '2029-12-15T12:34:56Z' '1.0.0' '[]' [ "$status" -eq 0 ] d=$(printf '%s' "$output" | jq -r '.scan_date') r=$(printf '%s' "$output" | jq -r '.repo') diff --git a/test/workflows/feature-ideation/fixtures/expected/empty-repo.signals.json b/test/workflows/feature-ideation/fixtures/expected/empty-repo.signals.json index 4101afb..e6438b5 100644 --- a/test/workflows/feature-ideation/fixtures/expected/empty-repo.signals.json +++ b/test/workflows/feature-ideation/fixtures/expected/empty-repo.signals.json @@ -1,6 +1,7 @@ { - "schema_version": "1.0.0", + "schema_version": "1.1.0", "scan_date": "2026-04-07T07:00:00Z", + "last_successful_run": "2026-03-31T07:00:00Z", "repo": "petry-projects/talkterm", "open_issues": { "count": 0, "items": [] }, "closed_issues_30d": { "count": 0, "items": [] }, diff --git a/test/workflows/feature-ideation/fixtures/expected/populated.signals.json b/test/workflows/feature-ideation/fixtures/expected/populated.signals.json index 4c92921..5618e36 100644 --- a/test/workflows/feature-ideation/fixtures/expected/populated.signals.json +++ b/test/workflows/feature-ideation/fixtures/expected/populated.signals.json @@ -1,6 +1,7 @@ { - "schema_version": "1.0.0", + "schema_version": "1.1.0", "scan_date": "2026-04-07T07:00:00Z", + "last_successful_run": "2026-03-31T07:00:00Z", "repo": "petry-projects/talkterm", "open_issues": { "count": 2, diff --git a/test/workflows/feature-ideation/fixtures/expected/truncated.signals.json b/test/workflows/feature-ideation/fixtures/expected/truncated.signals.json index 845db66..2c884ea 100644 --- a/test/workflows/feature-ideation/fixtures/expected/truncated.signals.json +++ b/test/workflows/feature-ideation/fixtures/expected/truncated.signals.json @@ -1,6 +1,7 @@ { - "schema_version": "1.0.0", + "schema_version": "1.1.0", "scan_date": "2026-04-07T07:00:00Z", + "last_successful_run": "2026-03-31T07:00:00Z", "repo": "petry-projects/talkterm", "open_issues": { "count": 0, "items": [] }, "closed_issues_30d": { "count": 0, "items": [] }, diff --git a/test/workflows/feature-ideation/fixtures/gh-responses/run-list-last-success.txt b/test/workflows/feature-ideation/fixtures/gh-responses/run-list-last-success.txt new file mode 100644 index 0000000..2ed9415 --- /dev/null +++ b/test/workflows/feature-ideation/fixtures/gh-responses/run-list-last-success.txt @@ -0,0 +1 @@ +2026-03-31T07:00:00Z \ No newline at end of file