Review your GitHub Copilot customization kit from your own scripts
Send the kit — .github/copilot-instructions.md,
.github/instructions/*.instructions.md with their applyTo globs,
.github/prompts/*.prompt.md, .github/chatmodes/*.chatmode.md,
AGENTS.md, .vscode/mcp.json, one file or several, each preceded by
a # file: .github/copilot-instructions.md marker line — and get back one
JSON object: a posture, the inventory of every instruction file, prompt, chat mode, rule,
glob, variable, tool and MCP server with its role, ranked findings across scope, frontmatter,
effectiveness, conflict, safety and hygiene, each with corrected frontmatter or Markdown you
can paste, the whole copilot-instructions.md rewritten, a reconciliation of every
prescan flag you send, quick wins, and the focus areas to work through first. Everything this
app does goes through the SkillSafe App API — plain JSON over HTTPS — so you can
hang a review off any pull request that touches .github/. Wire it into whatever
produces or reviews your Copilot customization: a pre-merge check on the
.github/instructions/ directory, a scheduled audit of every repo's instructions
file, or an editor command. Pick a language once and the whole page follows.
Basics
Base URL: https://api.skillsafe.ai/v1/app-api, app slug
copilot-clinic. Every request sends
Authorization: Bearer <token> and JSON bodies with
Content-Type: application/json. Responses are wrapped in an envelope:
{"data": …} on success, {"error": {"code", "message"}} on failure.
The review itself is produced by the gpt-terra model. Estimates are free;
runs are metered against your credit balance. There is a single run task — one pasted
kit in, one review out, no follow-up calls and no session state to carry.
Derived from
@github/awesome-copilot
(MIT), narrowed from a library of example customization files to reviewing one concrete
pasted kit. Not affiliated with GitHub.
| Status | Meaning |
|---|---|
401 | Missing or expired token — create a new session. |
402 | Not enough credits — top up at skillsafe.ai/account/credits. |
403 | The token isn't allowed to do this (e.g. a guest reviewing a very large kit). |
404 | Unknown job or record id. |
5xx | Transient platform error — retry with backoff. |
Browsers enforce CORS for this API, so run these examples from a server, script or terminal — not from another website's frontend.
Step 0 — A tiny client
Every task below is a single HTTP call, so start with a short helper that adds the auth
header, sends JSON and unwraps the data envelope. The later steps reuse it.
export API="https://api.skillsafe.ai/v1/app-api"
export TOKEN="YOUR_TOKEN" # see step 1
# every call looks like:
# curl -s "$API/..." -H "Authorization: Bearer $TOKEN" [-d '{json}']
# jq is used below to pull fields out of the {"data": ...} envelope
import json, requests
API = "https://api.skillsafe.ai/v1/app-api"
TOKEN = "YOUR_TOKEN" # see step 1 - read it from your shell environment in real code
def api(method, path, body=None, **headers):
res = requests.request(method, API + path, json=body,
headers={"Authorization": f"Bearer {TOKEN}", **headers})
payload = res.json()
if not res.ok:
raise RuntimeError(payload.get("error", {}).get("message", res.reason))
return payload["data"]
// Node 18+ (built-in fetch)
const API = "https://api.skillsafe.ai/v1/app-api";
const TOKEN = "YOUR_TOKEN"; // see step 1 - read it from your shell environment in real code
async function api(method, path, body, extraHeaders = {}) {
const res = await fetch(API + path, {
method,
headers: { Authorization: `Bearer ${TOKEN}`, "Content-Type": "application/json", ...extraHeaders },
body: body === undefined ? undefined : JSON.stringify(body),
});
const json = await res.json();
if (!res.ok) throw new Error(json.error?.message ?? res.statusText);
return json.data;
}
package main
import (
"bytes"
"encoding/json"
"fmt"
"net/http"
"os"
)
const API = "https://api.skillsafe.ai/v1/app-api"
var token = os.Getenv("SKILLSAFE_TOKEN") // see step 1
func call(method, path string, body, out any) error {
var buf bytes.Buffer
if body != nil {
json.NewEncoder(&buf).Encode(body)
}
req, _ := http.NewRequest(method, API+path, &buf)
req.Header.Set("Authorization", "Bearer "+token)
req.Header.Set("Content-Type", "application/json")
res, err := http.DefaultClient.Do(req)
if err != nil {
return err
}
defer res.Body.Close()
var env struct {
Data json.RawMessage `json:"data"`
Error *struct{ Message string `json:"message"` } `json:"error"`
}
json.NewDecoder(res.Body).Decode(&env)
if res.StatusCode >= 400 {
return fmt.Errorf("api %s %s: %s", method, path, env.Error.Message)
}
if out == nil {
return nil
}
return json.Unmarshal(env.Data, out)
}
// Java 17+, no dependencies. Pair with your JSON library (Jackson, Gson)
// to read fields out of the returned envelope.
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
public class SkillSafe {
static final String API = "https://api.skillsafe.ai/v1/app-api";
static final String TOKEN = System.getenv("SKILLSAFE_TOKEN"); // see step 1
static final HttpClient HTTP = HttpClient.newHttpClient();
static String api(String method, String path, String jsonBody) throws Exception {
var req = HttpRequest.newBuilder(URI.create(API + path))
.header("Authorization", "Bearer " + TOKEN)
.header("Content-Type", "application/json")
.method(method, jsonBody == null
? HttpRequest.BodyPublishers.noBody()
: HttpRequest.BodyPublishers.ofString(jsonBody))
.build();
var res = HTTP.send(req, HttpResponse.BodyHandlers.ofString());
if (res.statusCode() >= 400) throw new RuntimeException(res.body());
return res.body(); // envelope: {"data": ...}
}
}
require "net/http"
require "json"
API = "https://api.skillsafe.ai/v1/app-api"
TOKEN = ENV.fetch("SKILLSAFE_TOKEN") # see step 1
def api(method, path, body = nil)
uri = URI(API + path)
req = Net::HTTP.const_get(method.capitalize).new(uri)
req["Authorization"] = "Bearer #{TOKEN}"
req["Content-Type"] = "application/json"
req.body = body.to_json if body
res = Net::HTTP.start(uri.host, uri.port, use_ssl: true) { |h| h.request(req) }
payload = JSON.parse(res.body)
raise (payload.dig("error", "message") || res.message) unless res.is_a?(Net::HTTPSuccess)
payload["data"]
end
<?php
const API = "https://api.skillsafe.ai/v1/app-api";
$TOKEN = getenv("SKILLSAFE_TOKEN"); // see step 1
function api(string $method, string $path, ?array $body = null): mixed {
global $TOKEN;
$ch = curl_init(API . $path);
curl_setopt_array($ch, [
CURLOPT_CUSTOMREQUEST => $method,
CURLOPT_RETURNTRANSFER => true,
CURLOPT_HTTPHEADER => [
"Authorization: Bearer $TOKEN",
"Content-Type: application/json",
],
CURLOPT_POSTFIELDS => $body === null ? null : json_encode($body),
]);
$payload = json_decode(curl_exec($ch), true);
$status = curl_getinfo($ch, CURLINFO_RESPONSE_CODE);
curl_close($ch);
if ($status >= 400) {
throw new Exception($payload["error"]["message"] ?? "HTTP $status");
}
return $payload["data"];
}
// .NET 8+
using System.Net.Http.Json;
using System.Text.Json;
static class SkillSafe
{
const string Api = "https://api.skillsafe.ai/v1/app-api";
static readonly HttpClient Http = new();
static SkillSafe() =>
Http.DefaultRequestHeaders.Authorization =
new("Bearer", Environment.GetEnvironmentVariable("SKILLSAFE_TOKEN")); // see step 1
public static async Task<JsonElement> ApiAsync(HttpMethod method, string path, object? body = null)
{
var req = new HttpRequestMessage(method, Api + path);
if (body != null) req.Content = JsonContent.Create(body);
var res = await Http.SendAsync(req);
var json = await res.Content.ReadFromJsonAsync<JsonElement>();
if (!res.IsSuccessStatusCode)
throw new Exception(json.GetProperty("error").GetProperty("message").GetString());
return json.GetProperty("data");
}
}
Step 1 — Get a token
A guest token lets you check balances and estimate costs for free. For metered review runs
billed to your own account, use your personal token: open the
token page, sign in with SkillSafe, and press
Copy shell export — it puts export SKILLSAFE_TOKEN="…" on your
clipboard, which every example below reads. Treat the token like a password: it can spend
your credits. For fully headless scripts, POST /guest mints a guest token with
no browser involved.
curl -s -X POST "$API/guest" \
-H "Content-Type: application/json" \
-d '{"slug":"copilot-clinic"}' | jq -r '.data.token'
token = api("POST", "/guest", {"slug": "copilot-clinic"})["token"]
const { token } = await api("POST", "/guest", { slug: "copilot-clinic" });
var guest struct{ Token string `json:"token"` }
err := call("POST", "/guest", map[string]string{"slug": "copilot-clinic"}, &guest)
String envelope = api("POST", "/guest", """
{"slug":"copilot-clinic"}""");
// token is at data.token in the returned JSON
token = api("POST", "/guest", { slug: "copilot-clinic" })["token"]
$token = api("POST", "/guest", ["slug" => "copilot-clinic"])["token"];
var guest = await SkillSafe.ApiAsync(HttpMethod.Post, "/guest",
new { slug = "copilot-clinic" });
var token = guest.GetProperty("token").GetString();
The app stores this browser's token under the localStorage key
skillsafe_app_token:copilot-clinic, on the app's own origin. The
token page reads and manages it for you — you never need
to open developer tools.
Step 2 — Check who you are and your balance
Returns subject_type ("user" or "guest"),
subject_id and your credits balance. Check this before reviewing a
large kit.
curl -s "$API/me" -H "Authorization: Bearer $TOKEN" | jq '.data'
me = api("GET", "/me")
print(me["subject_type"], me["credits"])
const me = await api("GET", "/me");
console.log(me.subject_type, me.credits);
var me struct {
SubjectType string `json:"subject_type"`
Credits int64 `json:"credits"`
}
err := call("GET", "/me", nil, &me)
String envelope = api("GET", "/me", null);
// data.subject_type, data.credits
me = api("GET", "/me")
puts "#{me["subject_type"]}: #{me["credits"]} credits"
$me = api("GET", "/me");
echo "{$me['subject_type']}: {$me['credits']} credits\n";
var me = await SkillSafe.ApiAsync(HttpMethod.Get, "/me");
Console.WriteLine($"{me.GetProperty("subject_type")}: {me.GetProperty("credits")} credits");
Step 3 — Estimate the cost
Send exactly the input you would send to /run; the response's
hold_credits is the worst-case cost. Nothing is charged and no job is created,
so estimating is free — useful when you are piping a whole .github/
directory in and want a ceiling before spending credits.
| Input field | Type | Notes |
|---|---|---|
files | string, required | The pasted kit: .github/copilot-instructions.md, .github/instructions/*.instructions.md, .github/prompts/*.prompt.md, .github/chatmodes/*.chatmode.md, AGENTS.md, .vscode/mcp.json. One file or several concatenated, each preceded by a # file: .github/copilot-instructions.md marker line so the review can attribute every finding to the right file. This is the model's only evidence — nothing is executed, no MCP server is contacted and no editor is launched. Inputs longer than 100,000 characters are clipped middle-out, with a [... clipped ...] marker showing where. At least 60 characters are needed for a review. |
surface | string | mixed | instructions | prompt | chatmode | agents | mcp — what you pasted, which changes what counts as correct: an instruction file is judged on its applyTo scope and on whether its rules are specific enough to change a suggestion, a prompt file on its mode, model, tools and description frontmatter and on whether the task it describes is actually runnable, a chat mode on the persona and tool set it locks in, an AGENTS.md on whether it gives an agent the concrete commands it needs, an mcp.json on pinning, transport and credentials. |
concern | string | general | scope | effectiveness | conflicts | safety — the review emphasis. It weights the findings and the summary, but it is emphasis and not exclusivity: a high-severity finding from another category is never suppressed, so a token committed in .vscode/mcp.json is still called out under concern: "effectiveness". |
context | string, optional | Extra context: which editors and Copilot surfaces your team uses, how many people the kit is shared with, whether the repository is public, what your CI does with it, which instruction files are new versus long-standing, and any looseness you have already chosen to accept. Clipped at 20,000 characters. |
prescan_facts | object, optional | What a client-side scanner mechanically matched in the kit: {"resources": [], "flags": []}. Each entry is {id, label}. Resource ids look like res:instructionfile/-github-copilot-instructions.md, res:glob/**-*.tsx, res:rule/..., res:mcpserver/github or res:variable/${selection}; flag ids are <family>:<name>, drawn from twenty-six families — frontmatter-missing, frontmatter-broken, applyto-unquoted, applyto-missing, applyto-broad, applyto-overlap, glob-unmatchable, mode-missing, mode-invalid, tools-empty, description-missing, unknown-key, var-unknown, selection-implied, secret-literal, mcp-unpinned, mcp-plaintext, instructions-bloat, no-headings, vague-rule, persona-prose, duplicate-rule, conflict-rule, path-outside, agents-no-commands and no-global-instructions. Every flag id you send comes back in coverage_check. The web UI fills this from its own scan; API callers may omit the field or send the two empty arrays. |
retry_note | string, optional | Only set by the app's automatic reformat retry when a first reply was not valid JSON. Leave it out. |
cat > kit.md <<'KIT'
# file: .github/copilot-instructions.md
Always write clean, maintainable code and follow best practices.
Use pnpm for every package operation. Tests are written with vitest.
# file: .github/instructions/react.instructions.md
---
applyTo: **/*.tsx
---
Use interface for component props. Function components only, no class components.
KIT
jq -n --rawfile kit kit.md \
'{files: $kit,
surface: "mixed",
concern: "general",
context: "Public repo, eight contributors, Copilot in VS Code and on github.com.",
prescan_facts: {resources: [], flags: []}}' > input.json
curl -s -X POST "$API/estimate" \
-H "Authorization: Bearer $TOKEN" -H "Content-Type: application/json" \
-d @input.json | jq '.data.hold_credits'
KIT = """# file: .github/copilot-instructions.md
Always write clean, maintainable code and follow best practices.
Use pnpm for every package operation. Tests are written with vitest.
# file: .github/instructions/react.instructions.md
---
applyTo: **/*.tsx
---
Use interface for component props. Function components only, no class components.
"""
payload = {
"files": KIT,
"surface": "mixed",
"concern": "general",
"context": "Public repo, eight contributors, Copilot in VS Code and on github.com.",
"prescan_facts": {"resources": [], "flags": []},
}
est = api("POST", "/estimate", payload)
print("worst case:", est.get("hold_credits", est.get("credits")), "credits")
const kit = [
'# file: .github/copilot-instructions.md',
'Always write clean, maintainable code and follow best practices.',
'Use pnpm for every package operation. Tests are written with vitest.',
'',
'# file: .github/instructions/react.instructions.md',
'---',
'applyTo: **/*.tsx',
'---',
'Use interface for component props. Function components only, no class components.',
].join("\n");
const payload = {
files: kit,
surface: "mixed",
concern: "general",
context: "Public repo, eight contributors, Copilot in VS Code and on github.com.",
prescan_facts: { resources: [], flags: [] },
};
const est = await api("POST", "/estimate", payload);
console.log("worst case:", est.hold_credits ?? est.credits, "credits");
const kit = `# file: .github/copilot-instructions.md
Always write clean, maintainable code and follow best practices.
Use pnpm for every package operation. Tests are written with vitest.
# file: .github/instructions/react.instructions.md
---
applyTo: **/*.tsx
---
Use interface for component props. Function components only, no class components.`
payload := map[string]any{
"files": kit,
"surface": "mixed",
"concern": "general",
"context": "Public repo, eight contributors, Copilot in VS Code and on github.com.",
"prescan_facts": map[string]any{
"resources": []any{}, "flags": []any{},
},
}
var est struct{ HoldCredits int64 `json:"hold_credits"` }
err := call("POST", "/estimate", payload, &est)
String kit = """
# file: .github/copilot-instructions.md
Always write clean, maintainable code and follow best practices.
Use pnpm for every package operation. Tests are written with vitest.
# file: .github/instructions/react.instructions.md
---
applyTo: **/*.tsx
---
Use interface for component props. Function components only, no class components.
""";
String jsonPayload = """
{"files": %s,
"surface": "mixed",
"concern": "general",
"context": "Public repo, eight contributors, Copilot in VS Code and on github.com.",
"prescan_facts": {"resources": [], "flags": []}}
""".formatted(toJsonString(kit));
String envelope = api("POST", "/estimate", jsonPayload);
// worst-case cost is at data.hold_credits
KIT = <<~FILES
# file: .github/copilot-instructions.md
Always write clean, maintainable code and follow best practices.
Use pnpm for every package operation. Tests are written with vitest.
# file: .github/instructions/react.instructions.md
---
applyTo: **/*.tsx
---
Use interface for component props. Function components only, no class components.
FILES
payload = { files: KIT,
surface: "mixed",
concern: "general",
context: "Public repo, eight contributors, Copilot in VS Code and on github.com.",
prescan_facts: { resources: [], flags: [] } }
est = api("POST", "/estimate", payload)
puts "worst case: #{est["hold_credits"] || est["credits"]} credits"
$kit = <<<'KIT'
# file: .github/copilot-instructions.md
Always write clean, maintainable code and follow best practices.
Use pnpm for every package operation. Tests are written with vitest.
# file: .github/instructions/react.instructions.md
---
applyTo: **/*.tsx
---
Use interface for component props. Function components only, no class components.
KIT;
$payload = [
"files" => $kit,
"surface" => "mixed",
"concern" => "general",
"context" => "Public repo, eight contributors, Copilot in VS Code and on github.com.",
"prescan_facts" => ["resources" => [], "flags" => []],
];
$est = api("POST", "/estimate", $payload);
echo "worst case: " . ($est["hold_credits"] ?? $est["credits"]) . " credits\n";
var kit = """
# file: .github/copilot-instructions.md
Always write clean, maintainable code and follow best practices.
Use pnpm for every package operation. Tests are written with vitest.
# file: .github/instructions/react.instructions.md
---
applyTo: **/*.tsx
---
Use interface for component props. Function components only, no class components.
""";
var payload = new {
files = kit,
surface = "mixed",
concern = "general",
context = "Public repo, eight contributors, Copilot in VS Code and on github.com.",
prescan_facts = new {
resources = Array.Empty<object>(), flags = Array.Empty<object>(),
},
};
var est = await SkillSafe.ApiAsync(HttpMethod.Post, "/estimate", payload);
Console.WriteLine($"worst case: {est.GetProperty("hold_credits")} credits");
prescan_facts.flags is how you make the review answer for things you already
know about. Send {"resources": [{"id": "res:glob/**-*.tsx", "label": "Glob/**/*.tsx"}],
"flags": [{"id": "applyto-unquoted:react.instructions.md", "label": "applyTo glob is not quoted"}]} and every flag id
comes back in coverage_check — addressed by a finding, or set aside with
the reason. Nothing you flag is silently dropped, which makes it the field to assert on in a
CI check.
Step 4 — Run the review and wait for the result
/run takes the same input as /estimate, places a credit hold and
returns a job_id. Poll /jobs/{job_id} every 1–2 seconds
until status is succeeded or failed (a run typically
takes 30–90 s, since every finding carries corrected frontmatter or Markdown and the
reply also rewrites the whole instructions file). Always send an Idempotency-Key
header so a network retry can't start a second, double-charged run — the app sends one on
every run, and its automatic reformat retry reuses a key derived from the same input. The
review is in output — usually nested as output.output, and as a
JSON string, so parse defensively. The samples below print the posture, the inventory,
the ranked findings and the focus areas, save the whole object to review.json,
and write corrected_instructions out as a Markdown file you can drop into
.github/.
JOB_ID=$(curl -s -X POST "$API/run" \
-H "Authorization: Bearer $TOKEN" -H "Content-Type: application/json" \
-H "Idempotency-Key: cc-$(date +%s)" \
-d @input.json | jq -r '.data.job_id')
while :; do
JOB=$(curl -s "$API/jobs/$JOB_ID" -H "Authorization: Bearer $TOKEN")
STATUS=$(echo "$JOB" | jq -r '.data.status')
[ "$STATUS" = "succeeded" ] || [ "$STATUS" = "failed" ] && break
sleep 2
done
# unwrap the review once, then read it
echo "$JOB" | jq -r '.data.output.output' > review.json
jq -r '
"\(.review_name) [\(.posture)]: \(.verdict)",
"",
"INVENTORY",
(.inventory[] | " \(.kind)/\(.name) in \(.scope) - \(.role)"),
"",
"FINDINGS",
(.findings[] | " [\(.priority)] \(.id) \(.category) \(.resource): \(.problem)"),
"",
"QUICK WINS",
(.quick_wins[] | " - \(.)"),
"",
"FOCUS AREAS",
(.focus_areas[] | " \(.area) - \(.why)"),
"",
"COVERAGE",
(.coverage_check[] | " \(.id): \(if .addressed then "ok" else "SET ASIDE" end) - \(.note)")' \
review.json
# the headline artifact: the whole copilot-instructions.md, rewritten and ready to commit
jq -r 'select(.corrected_instructions != "") | .corrected_instructions' review.json \
> copilot-instructions.new.md
[ -s copilot-instructions.new.md ] && \
mv copilot-instructions.new.md .github/copilot-instructions.md
# fail the pipeline on anything critical
jq -e '[.findings[] | select(.priority == "critical")] | length == 0' review.json > /dev/null \
|| { echo "critical findings present"; exit 1; }
import time
job_id = api("POST", "/run", payload,
**{"Idempotency-Key": "cc-001"})["job_id"]
while True:
job = api("GET", f"/jobs/{job_id}")
if job["status"] in ("succeeded", "failed"):
break
time.sleep(1.5)
if job["status"] == "failed":
raise RuntimeError(job.get("error", "run failed"))
raw = job["output"]
if isinstance(raw, dict) and "output" in raw:
raw = raw["output"]
review = json.loads(raw) if isinstance(raw, str) else raw
print(f'{review["review_name"]} [{review["posture"]}]: {review["verdict"]}')
for r in review["inventory"]:
print(f' {r["kind"]}/{r["name"]:<28} in={r["scope"] or "-":<24} {r["role"]}')
for f in review["findings"]:
print(f' [{f["priority"]:>8}] {f["id"]} {f["category"]} {f["resource"]}')
print(f' L:{f["likelihood"]}/S:{f["severity"]} {f["problem"]}')
print(f' fix: {f["fix"]}')
if f["snippet"]:
print(" snippet:", f["snippet"].splitlines()[0], "...")
for w in review["quick_wins"]:
print(" win:", w)
for a in review["focus_areas"]:
print(f' focus {a["area"]} {a["finding_ids"]} - {a["why"]}')
for c in review["coverage_check"]:
print(f' {c["id"]}: {"ok" if c["addressed"] else "SET ASIDE"} - {c["note"]}')
with open("review.json", "w", encoding="utf-8") as fh:
json.dump(review, fh, indent=2)
# the headline artifact: Markdown, so read the diff before you commit it
if review["corrected_instructions"]:
with open("copilot-instructions.new.md", "w", encoding="utf-8") as fh:
fh.write(review["corrected_instructions"])
critical = [f for f in review["findings"] if f["priority"] == "critical"]
if critical:
raise SystemExit(f"{len(critical)} critical finding(s)")
import { writeFileSync } from "node:fs";
const { job_id } = await api("POST", "/run", payload,
{ "Idempotency-Key": crypto.randomUUID() });
let job;
do {
await new Promise((r) => setTimeout(r, 1500));
job = await api("GET", `/jobs/${job_id}`);
} while (job.status !== "succeeded" && job.status !== "failed");
if (job.status === "failed") throw new Error(job.error ?? "run failed");
const raw = job.output?.output ?? job.output;
const review = typeof raw === "string" ? JSON.parse(raw) : raw;
console.log(`${review.review_name} [${review.posture}]: ${review.verdict}`);
for (const r of review.inventory) {
console.log(` ${r.kind}/${r.name} (${r.scope || "-"}): ${r.role}`);
}
for (const f of review.findings) {
console.log(` [${f.priority}] ${f.id} ${f.category} ${f.resource}`);
console.log(` L:${f.likelihood}/S:${f.severity} - ${f.fix}`);
}
for (const w of review.quick_wins) console.log(` win: ${w}`);
for (const a of review.focus_areas) {
console.log(` focus ${a.area} (${a.finding_ids.join(", ")}): ${a.why}`);
}
for (const c of review.coverage_check) {
console.log(` ${c.id}: ${c.addressed ? "ok" : "SET ASIDE"} - ${c.note}`);
}
writeFileSync("review.json", JSON.stringify(review, null, 2));
// the headline artifact: a copilot-instructions.md you can commit as-is
if (review.corrected_instructions) {
writeFileSync("copilot-instructions.new.md", review.corrected_instructions);
}
const critical = review.findings.filter((f) => f.priority === "critical");
if (critical.length) process.exitCode = 1;
var started struct{ JobID string `json:"job_id"` }
if err := call("POST", "/run", payload, &started); err != nil {
log.Fatal(err)
}
var job struct {
Status string `json:"status"`
Error string `json:"error"`
Output json.RawMessage `json:"output"`
}
for {
if err := call("GET", "/jobs/"+started.JobID, nil, &job); err != nil {
log.Fatal(err)
}
if job.Status == "succeeded" || job.Status == "failed" {
break
}
time.Sleep(1500 * time.Millisecond)
}
// job.Output is {"output": "<json string>"} - unwrap, then unmarshal:
type Review struct {
ReviewName string `json:"review_name"`
Posture string `json:"posture"`
Verdict string `json:"verdict"`
ExecSummary string `json:"exec_summary"`
Assumptions []string `json:"assumptions"`
OpenQuestions []string `json:"open_questions"`
Inventory []struct {
Kind, Name, Scope, Role string
} `json:"inventory"`
Findings []struct {
ID, Category, Severity, Likelihood, Priority string
Resource, Problem, Impact, Fix, Snippet string
} `json:"findings"`
CoverageCheck []struct {
ID, Note string
Addressed bool
} `json:"coverage_check"`
CorrectedInstructions string `json:"corrected_instructions"`
QuickWins []string `json:"quick_wins"`
FocusAreas []struct {
Area, Why string
FindingIDs []string `json:"finding_ids"`
} `json:"focus_areas"`
Summary string `json:"summary"`
}
var wrapper struct{ Output string `json:"output"` }
json.Unmarshal(job.Output, &wrapper)
var review Review
json.Unmarshal([]byte(wrapper.Output), &review)
fmt.Printf("%s [%s]: %s\n", review.ReviewName, review.Posture, review.Verdict)
for _, r := range review.Inventory {
fmt.Printf(" %s/%s (%s): %s\n", r.Kind, r.Name, r.Scope, r.Role)
}
for _, f := range review.Findings {
fmt.Printf(" [%s] %s %s %s: %s\n", f.Priority, f.ID, f.Category, f.Resource, f.Problem)
}
for _, a := range review.FocusAreas {
fmt.Printf(" focus %s %v: %s\n", a.Area, a.FindingIDs, a.Why)
}
os.WriteFile("review.json", []byte(wrapper.Output), 0o644)
// the headline artifact: the rewritten instructions file, as Markdown
if review.CorrectedInstructions != "" {
os.WriteFile("copilot-instructions.new.md", []byte(review.CorrectedInstructions), 0o644)
}
String envelope = api("POST", "/run", jsonPayload);
String jobId = /* data.job_id via your JSON library */;
while (true) {
String job = api("GET", "/jobs/" + jobId, null);
String status = /* data.status */;
if (status.equals("succeeded") || status.equals("failed")) break;
Thread.sleep(1500);
}
// The review is at data.output.output as a JSON string - parse it again, then read
// review_name, posture, verdict, exec_summary, assumptions[], open_questions[],
// inventory[] (kind/name/scope/role),
// findings[] (id/category/severity/likelihood/priority/resource/problem/impact/fix/snippet),
// coverage_check[] (id/addressed/note), corrected_instructions, quick_wins[],
// focus_areas[] (area/why/finding_ids[]) and summary.
// Finally keep the review on disk, and write the rewritten instructions file:
// Files.writeString(Path.of("review.json"), reviewJson);
// if (!correctedInstructions.isEmpty())
// Files.writeString(Path.of("copilot-instructions.new.md"), correctedInstructions);
started = api("POST", "/run", payload)
job = nil
loop do
job = api("GET", "/jobs/#{started["job_id"]}")
break if %w[succeeded failed].include?(job["status"])
sleep 1.5
end
raise (job["error"] || "run failed") if job["status"] == "failed"
raw = job["output"].is_a?(Hash) ? job["output"].fetch("output", job["output"]) : job["output"]
review = raw.is_a?(String) ? JSON.parse(raw) : raw
puts "#{review["review_name"]} [#{review["posture"]}]: #{review["verdict"]}"
review["inventory"].each { |r| puts " #{r["kind"]}/#{r["name"]} (#{r["scope"]}): #{r["role"]}" }
review["findings"].each do |f|
puts " [#{f["priority"]}] #{f["id"]} #{f["category"]} #{f["resource"]}"
puts " L:#{f["likelihood"]}/S:#{f["severity"]} - #{f["fix"]}"
end
review["quick_wins"].each { |w| puts " win: #{w}" }
review["focus_areas"].each { |a| puts " focus #{a["area"]} #{a["finding_ids"].join(", ")}" }
review["coverage_check"].each { |c| puts " #{c["id"]}: #{c["addressed"] ? "ok" : "SET ASIDE"}" }
File.write("review.json", JSON.pretty_generate(review))
# the headline artifact
unless review["corrected_instructions"].to_s.empty?
File.write("copilot-instructions.new.md", review["corrected_instructions"])
end
exit 1 if review["findings"].any? { |f| f["priority"] == "critical" }
$started = api("POST", "/run", $payload);
do {
sleep(2);
$job = api("GET", "/jobs/" . $started["job_id"]);
} while (!in_array($job["status"], ["succeeded", "failed"]));
if ($job["status"] === "failed") {
throw new Exception($job["error"] ?? "run failed");
}
$raw = is_array($job["output"]) ? ($job["output"]["output"] ?? $job["output"]) : $job["output"];
$review = is_string($raw) ? json_decode($raw, true) : $raw;
echo "{$review['review_name']} [{$review['posture']}]: {$review['verdict']}\n";
foreach ($review["inventory"] as $r) {
echo " {$r['kind']}/{$r['name']} ({$r['scope']}): {$r['role']}\n";
}
foreach ($review["findings"] as $f) {
echo " [{$f['priority']}] {$f['id']} {$f['category']} {$f['resource']}\n";
echo " L:{$f['likelihood']}/S:{$f['severity']} - {$f['fix']}\n";
}
foreach ($review["quick_wins"] as $w) {
echo " win: $w\n";
}
foreach ($review["focus_areas"] as $a) {
echo " focus {$a['area']}: " . implode(", ", $a["finding_ids"]) . "\n";
}
foreach ($review["coverage_check"] as $c) {
echo " {$c['id']}: " . ($c["addressed"] ? "ok" : "SET ASIDE") . "\n";
}
file_put_contents("review.json", json_encode($review, JSON_PRETTY_PRINT));
// the headline artifact
if (($review["corrected_instructions"] ?? "") !== "") {
file_put_contents("copilot-instructions.new.md", $review["corrected_instructions"]);
}
var started = await SkillSafe.ApiAsync(HttpMethod.Post, "/run", payload);
var jobId = started.GetProperty("job_id").GetString();
JsonElement job;
while (true)
{
job = await SkillSafe.ApiAsync(HttpMethod.Get, $"/jobs/{jobId}");
var status = job.GetProperty("status").GetString();
if (status is "succeeded" or "failed") break;
await Task.Delay(1500);
}
var rawText = job.GetProperty("output").GetProperty("output").GetString();
using var doc = JsonDocument.Parse(rawText!);
var review = doc.RootElement;
Console.WriteLine($"{review.GetProperty("review_name")} " +
$"[{review.GetProperty("posture")}]: {review.GetProperty("verdict")}");
foreach (var r in review.GetProperty("inventory").EnumerateArray())
{
Console.WriteLine($" {r.GetProperty("kind")}/{r.GetProperty("name")}: {r.GetProperty("role")}");
}
foreach (var f in review.GetProperty("findings").EnumerateArray())
{
Console.WriteLine($" [{f.GetProperty("priority")}] {f.GetProperty("id")} " +
$"{f.GetProperty("category")} {f.GetProperty("resource")} " +
$"(L:{f.GetProperty("likelihood")}/S:{f.GetProperty("severity")})");
}
foreach (var a in review.GetProperty("focus_areas").EnumerateArray())
{
Console.WriteLine($" focus {a.GetProperty("area")}: {a.GetProperty("why")}");
}
await File.WriteAllTextAsync("review.json", rawText!);
// the headline artifact
var rewritten = review.GetProperty("corrected_instructions").GetString();
if (!string.IsNullOrEmpty(rewritten))
{
await File.WriteAllTextAsync("copilot-instructions.new.md", rewritten);
}
The model is asked for one JSON object and nothing else, but a stray code fence or preamble
is always possible. Strip a leading ```json fence, take the text between the
first { and the last }, and only then parse — that is what
the app does before it falls back to a retry_note reformat run.
The review object — output schema
One JSON object, always the same shape. Every array is present, and the review is grounded in
the pasted kit alone: findings cite only files, rules, globs, variables and servers that
actually appear in files, and something that is simply absent (no
repository instructions file, no applyTo, no mode on a prompt file,
no commands in AGENTS.md) is reported against the nearest real file or against
(missing from the kit). Where the kit is silent on something that changes the
verdict you get an entry in assumptions and, if it would change the ranking, in
open_questions. Expect five to fifteen findings on a typical kit — a
carefully built one may honestly yield two or three, and findings is never empty.
| Field | Type | Meaning |
|---|---|---|
review_name | string | A short title naming the kit, taken from the kit's own naming — e.g. acme-web Copilot kit — customization review. |
posture | string | ready-to-commit | tighten-first | not-working-as-written. See the table below. |
verdict | string | One sentence justifying the posture and naming the single most important change. |
exec_summary | string | Two or three paragraphs, separated by blank lines, on the dominant themes across the kit. |
assumptions | string[] | Explicit assumptions filling gaps the kit left open. Read these first — a wrong assumption invalidates the findings built on it. |
open_questions | string[] | Questions whose answers would change the ranking. |
inventory | array | {kind, name, scope, role} — every InstructionFile, PromptFile, ChatMode, AgentsFile, McpServer, Rule, Glob, Variable and Tool the review parsed out of the paste and the part it plays. scope is where it applies — the applyTo glob, the file it is declared in, or the whole repository. |
findings | array | The ranked findings table — ids CP-001, CP-002, … in sequence, at least one entry. Columns are listed below. |
coverage_check | array | {id, addressed, note} — one entry per prescan_facts.flags id you sent, each appearing exactly once. See the semantics below. |
corrected_instructions | string | The headline artifact. The whole rewritten .github/copilot-instructions.md as Markdown — not a diff and not a fragment — carrying every fix the findings call for: persona padding cut, unenforceable phrasing made falsifiable, duplicated rules merged, contradictions resolved, and the surviving rules grouped under headings. Write it straight to .github/copilot-instructions.md. Empty string ("") when no repository instructions file was pasted — a paste of only instruction files, prompts or an mcp.json gets its corrections in the per-finding snippet fields instead. The web app measures the returned Markdown in your browser: word count, section count and how much unenforceable phrasing survived. |
quick_wins | string[] | One-line changes worth doing immediately, ahead of any planning. May be empty when nothing here is a one-liner. |
focus_areas | array | {area, why, finding_ids} — what to work through first, one sentence tied to the review, and the finding ids that motivate it. Every id in finding_ids exists in findings. |
summary | string | Closing paragraph: what to fix first, and what risk remains after that. |
The three posture values:
| posture | What it means |
|---|---|
ready-to-commit | The files parse, the globs are scoped and quoted, and the rules are specific and non-contradictory. Findings still exist, but they are additions rather than blockers — a rule worth splitting out into its own instruction file, a description worth sharpening, a prompt worth pinning to a model. Genuinely well-built kits land here rather than having severity manufactured for them. |
tighten-first | The shape is right, but named gaps should close before the kit is shared: an over-broad glob that applies frontend rules to the whole repository, unenforceable rules that no reviewer could check, the same rule duplicated across two files, a prompt file with no description. |
not-working-as-written | At least one file does not do what its author believes: broken or missing frontmatter, a scope that matches nothing in the repository, an agent prompt stuck in ask mode, contradictory rules on overlapping globs, or a credential committed in .vscode/mcp.json. |
Each entry in findings:
| Column | Meaning |
|---|---|
id | Sequential CP-001, CP-002, … — the stable handle referenced from focus_areas[].finding_ids. |
category | scope | frontmatter | effectiveness | conflict | safety | hygiene. Weighted by the concern you sent, but never restricted to it. |
severity | low | medium | high — how bad it is when it bites. |
likelihood | low | medium | high — how likely it is to bite. |
priority | critical | high | medium | low — severity by likelihood. critical is reserved for a file that silently does nothing it claims to do, a rule set that contradicts itself on every file it touches, or a committed credential, so sort on this field and work top-down. This is also the field to gate a pipeline on. |
resource | The InstructionFile/.github/instructions/react.instructions.md, Glob/**/*.tsx, PromptFile/new-endpoint.prompt.md, McpServer/github or rule this is about — always something that appears in files, or the literal (missing from the kit) when the finding is about something absent. |
problem | What is wrong, in this kit specifically. |
impact | What Copilot actually does because of it, and what that costs the team. |
fix | The concrete change to make — the file, the key, the glob, the wording — not "improve your instructions". |
snippet | A corrected frontmatter, Markdown or JSON fragment you can paste: the fixed block, correctly indented, matching the file it belongs to, not the whole file (that is corrected_instructions). Empty string when a snippet would add nothing. Secret values are never echoed — an environment reference or a placeholder appears instead. |
coverage_check semantics:
| Case | What you get |
|---|---|
| Every flag id you sent | Each prescan_facts.flags id appears in coverage_check exactly once. Nothing you flagged is silently dropped, which makes this the field to assert on in a CI check. Ids in prescan_facts.resources are not reconciled here — they shape the inventory instead. |
addressed: true | The flag is covered by the review; note names the finding id that covers it. |
addressed: false | The flag was deliberately set aside; note gives the reason — a check that fired but is not a real problem for this kit (a broad applyTo on a repository where every source file really is TypeScript, a repeated rule that reads as deliberate emphasis in two files with disjoint scopes). |
| Nothing sent | Omit prescan_facts, or send the two empty arrays, and coverage_check comes back empty. The rest of the review is unaffected. |
A small, realistic result for the snippet above, trimmed for length:
{
"review_name": "acme-web Copilot kit - customization review",
"posture": "not-working-as-written",
"verdict": "The React instruction file never loads because its applyTo glob is unquoted YAML, and the repository instructions contradict it anyway - quote the glob and settle the interface-versus-type rule before anything else in this kit is worth tuning.",
"exec_summary": "Two files in this kit do not do what their authors believe. react.instructions.md opens its frontmatter with an unquoted applyTo glob, which YAML reads as an alias node rather than a string, so the block fails to parse and the file contributes nothing to any suggestion. new-endpoint.prompt.md declares no mode, so it runs as an ask-mode prompt while its body instructs the model to create files, run pnpm gen:api and open a pull request. Both failures are silent: nothing warns you, the suggestions simply stay generic.\n\nThe second theme is contradiction. The repository instructions prefer type aliases for object shapes; the React file requires interface for component props. Both claim the same .tsx files, and the model resolves the clash differently from one completion to the next, which is why reviewers see the convention flip inside a single pull request.\n\nThe third is weight. copilot-instructions.md is roughly 700 words, opens with four sentences of persona, and repeats the vitest rule twice. The parts that actually change output - pnpm over npm, the generated directory being off limits, the repository layer in front of drizzle - are buried among instructions like 'write clean code' and 'handle errors properly' that no reviewer could ever check. A credential is also committed in .vscode/mcp.json, which is a separate and more urgent problem.",
"assumptions": [
"The repository is a pnpm workspace as the instructions state, since no package.json was pasted.",
"GITHUB_PERSONAL_ACCESS_TOKEN in .vscode/mcp.json is a live token, since it is a literal ghp_ value rather than an input reference.",
"apps/web is the only place .tsx files live, so the React glob is broad in form but narrow in practice."
],
"open_questions": [
"Is .vscode/mcp.json committed, or ignored and local to each machine? That decides whether the token has already leaked.",
"Is the interface rule for props deliberate, or a leftover from before the type-alias convention was adopted?"
],
"inventory": [
{ "kind": "InstructionFile", "name": ".github/copilot-instructions.md", "scope": "whole repository",
"role": "Repository-wide instructions: stack, package manager, test runner, layering rules, plus persona prose." },
{ "kind": "InstructionFile", "name": ".github/instructions/react.instructions.md", "scope": "**/*.tsx (declared, never applied)",
"role": "React component conventions; its frontmatter does not parse, so the rules never reach the model." },
{ "kind": "Glob", "name": "**/*.tsx", "scope": "react.instructions.md",
"role": "Intended scope for the React rules, written unquoted and therefore invalid YAML." },
{ "kind": "PromptFile", "name": ".github/prompts/new-endpoint.prompt.md", "scope": "invoked as /new-endpoint",
"role": "Seven-step scaffold for a REST endpoint; has a description but no mode, model or tools." },
{ "kind": "McpServer", "name": "github", "scope": ".vscode/mcp.json",
"role": "GitHub MCP server started through npx, unpinned, carrying a literal access token." },
{ "kind": "Rule", "name": "Use pnpm for every package operation", "scope": "whole repository",
"role": "The strongest rule in the kit: specific, checkable and tied to a real CI failure." },
{ "kind": "AgentsFile", "name": "AGENTS.md", "scope": "whole repository",
"role": "General working advice for agents; names no build, test or lint command." }
],
"findings": [
{ "id": "CP-001", "category": "frontmatter",
"severity": "high", "likelihood": "high", "priority": "critical",
"resource": "InstructionFile/.github/instructions/react.instructions.md",
"problem": "applyTo: **/*.tsx is unquoted. A YAML scalar beginning with an asterisk is an alias node, so the frontmatter block fails to parse.",
"impact": "The whole file is skipped. Every React rule in it - props typing, no class components, query hooks, aria-label on icon buttons - has been silently absent from suggestions since the file was added.",
"fix": "Quote the glob and add a description so the file is identifiable in the Copilot UI.",
"snippet": "---\napplyTo: \"**/*.tsx\"\ndescription: \"React component conventions for apps/web.\"\n---" },
{ "id": "CP-002", "category": "safety",
"severity": "high", "likelihood": "high", "priority": "critical",
"resource": "McpServer/github",
"problem": "A literal ghp_ personal access token is written into the env block of .vscode/mcp.json instead of being requested as an input.",
"impact": "If this file is committed the token is in git history, in every clone and in every fork. Deleting the line does not undo it: the token has to be revoked.",
"fix": "Revoke the token now, then declare an input in mcp.json so VS Code prompts for it and stores it in the secret store.",
"snippet": "{\n \"inputs\": [\n { \"id\": \"github_pat\", \"type\": \"promptString\", \"description\": \"GitHub PAT\", \"password\": true }\n ],\n \"servers\": {\n \"github\": {\n \"command\": \"npx\",\n \"args\": [\"-y\", \"@modelcontextprotocol/server-github@0.6.2\"],\n \"env\": { \"GITHUB_PERSONAL_ACCESS_TOKEN\": \"${input:github_pat}\" }\n }\n }\n}" },
{ "id": "CP-003", "category": "conflict",
"severity": "medium", "likelihood": "high", "priority": "high",
"resource": "Rule/interface versus type alias",
"problem": "copilot-instructions.md prefers type aliases for object shapes; react.instructions.md requires interface for component props. Both target the same .tsx files.",
"impact": "Once CP-001 is fixed and the React file starts loading, the model gets two rules for one decision and picks differently between completions, so the convention flips inside a single pull request.",
"fix": "Decide once and state the exception explicitly in the repository file, then delete the competing sentence from the other file.",
"snippet": "## TypeScript\n\n- Use a type alias for object shapes.\n- Exception: React component props use an interface named ComponentNameProps." },
{ "id": "CP-004", "category": "frontmatter",
"severity": "medium", "likelihood": "high", "priority": "high",
"resource": "PromptFile/.github/prompts/new-endpoint.prompt.md",
"problem": "The prompt has a description but no mode, so it runs in ask mode while its body tells the model to add route modules, run pnpm gen:api, run the test suite and open a pull request.",
"impact": "The prompt answers with a plan instead of doing the work, and the seven numbered steps read as a failure every time someone runs it.",
"fix": "Set mode: agent, name the tools the steps actually need, and pin a model so the prompt behaves the same for everyone.",
"snippet": "---\nmode: agent\nmodel: GPT-4.1\ntools: [\"codebase\", \"editFiles\", \"runCommands\"]\ndescription: \"Scaffold a new REST endpoint end to end.\"\n---" },
{ "id": "CP-005", "category": "effectiveness",
"severity": "medium", "likelihood": "medium", "priority": "medium",
"resource": "InstructionFile/.github/copilot-instructions.md",
"problem": "Roughly 700 words, four opening sentences of persona, one heading, and eleven unenforceable rules such as write clean code, follow best practices and handle errors properly. The vitest rule appears twice, verbatim.",
"impact": "The rules that would actually change a suggestion compete for attention with sentences that cannot be checked, and every request pays for the whole file.",
"fix": "Cut the persona and the unenforceable lines, merge the duplicated vitest rule, and group what survives under headings. corrected_instructions is the rewritten file.",
"snippet": "" },
{ "id": "CP-006", "category": "hygiene",
"severity": "low", "likelihood": "medium", "priority": "low",
"resource": "McpServer/github",
"problem": "npx -y @modelcontextprotocol/server-github resolves to whatever version is newest at launch, with install prompts suppressed.",
"impact": "The code that receives your token changes without any change to the repository, and two engineers can be running different builds on the same day.",
"fix": "Pin an exact version in args, as the snippet in CP-002 already does.",
"snippet": "" }
],
"coverage_check": [
{ "id": "applyto-unquoted:react.instructions.md", "addressed": true, "note": "CP-001." },
{ "id": "secret-literal:.vscode/mcp.json", "addressed": true, "note": "CP-002." },
{ "id": "conflict-rule:interface-vs-type-alias", "addressed": true, "note": "CP-003." },
{ "id": "mode-missing:new-endpoint.prompt.md", "addressed": true, "note": "CP-004." },
{ "id": "duplicate-rule:vitest", "addressed": true, "note": "CP-005, merged in the rewrite." },
{ "id": "mcp-unpinned:github", "addressed": true, "note": "CP-006." },
{ "id": "applyto-broad:testing.instructions.md", "addressed": false, "note": "Set aside: applyTo \"**\" is wide, but the rules in that file are about how tests are written and read correctly against any file the model touches." }
],
"corrected_instructions": "# Copilot instructions for acme-web\n\n## Stack\n\n- pnpm workspace. Shared libraries live in packages/, the API server in services/api, the frontend in apps/web.\n- Frontend: Vite, React 18, TanStack Query. Server state belongs in TanStack Query, local UI state in useState. Do not add Redux.\n- API: drizzle behind the repository layer in services/api/src/repositories. Route handlers never import the drizzle client directly.\n\n## Commands\n\n- Install from the workspace root with pnpm install. Never run npm install or yarn: it rewrites pnpm-lock.yaml and breaks CI.\n- Test with pnpm test (vitest). Do not add jest, a jest config, or imports from @jest/globals.\n- Regenerate API types with pnpm gen:api after editing openapi/schema.yaml.\n- Create migrations with pnpm db:migrate:new. Never hand-edit a migration that has merged to main.\n\n## Do not edit\n\n- src/generated/ is produced by the OpenAPI generator and is overwritten by pnpm gen:api. Fix openapi/schema.yaml instead.\n\n## TypeScript\n\n- Use a type alias for object shapes. Exception: React component props use an interface named ComponentNameProps.\n\n## Tests\n\n- Unit tests sit next to the file they cover and end in .test.ts.\n- Integration tests live in tests/integration and end in .spec.ts.\n\n## Logging and configuration\n\n- Use pino. console.log is allowed only in scripts/.\n- Every API log line carries the request id from async local storage.\n- Configuration is read once in services/api/src/config.ts and exported as one typed object. Do not read environment variables anywhere else.\n\n## Feature flags\n\n- Flags are declared in packages/flags/src/flags.ts and each needs an owner and a removal date in its comment, or the lint rule fails.\n",
"quick_wins": [
"Quote the applyTo glob in react.instructions.md - one pair of quotes turns the whole file back on.",
"Revoke the token in .vscode/mcp.json today, then add it to .gitignore or replace it with an input.",
"Add mode: agent to new-endpoint.prompt.md so the prompt can do the work it describes."
],
"focus_areas": [
{ "area": "Make the files load at all",
"why": "An instruction file with broken frontmatter and a prompt stuck in ask mode both fail silently, so no amount of rule tuning shows up until they are fixed.",
"finding_ids": ["CP-001", "CP-004"] },
{ "area": "Get the credential out of the repository",
"why": "A literal token in a committed mcp.json is already exposed; only revocation closes it.",
"finding_ids": ["CP-002"] },
{ "area": "Say each thing once, and say it checkably",
"why": "One contradiction and a page of unenforceable prose are what make the kit feel like it does nothing.",
"finding_ids": ["CP-003", "CP-005"] }
],
"summary": "Revoke the GitHub token, quote the applyTo glob, and add mode: agent to the prompt file - those three edits take minutes and turn two dead files back on. Then commit corrected_instructions: it drops the persona, merges the duplicated vitest rule, resolves the interface-versus-type conflict in one place and groups the surviving rules under headings, at roughly half the original length. After that the remaining risk is drift - the commands and paths in this file are only useful while they are true, so review it whenever the workspace layout changes."
}
corrected_instructions is the artifact to automate on: it is a whole file, so a
pipeline can write it to .github/copilot-instructions.md and open a pull request
with the diff. It is Markdown, not JSON — a plain string in the reply, so there is
nothing to parse, but there is also nothing that will fail loudly if it is wrong. Read the
diff and the findings that produced it, because cutting a rule that reads as unenforceable
can remove a convention someone relied on.
This is AI-generated review from pasted text, not a guarantee about what Copilot will do: it
sees only what you sent, never your editor, your repository or a real completion. Check
assumptions and open_questions before you act on the rankings,
validate every snippet and the rewritten instructions file before committing them, and keep a
human reviewer in the loop.
Step 5 — Stream the review as it is written
/run-stream takes exactly the same body as /run but answers with
server-sent events, so you can show progress instead of a spinner — useful here
because a full findings table plus a rewritten instructions file makes for a long reply. This
app's own progress panel is this endpoint. Events are separated by a blank line; each has an
event: line and a data: line carrying JSON.
| Event | Payload | Meaning |
|---|---|---|
job | {job_id, status} | Sent once, when the job is accepted — show "starting". |
delta | {text} | A chunk of the reply, in order. Append it; the accumulated length is your only progress signal (the total is not known in advance). The app advances its step list by watching for the "review_name", "inventory", "findings", "coverage_check", "corrected_instructions" and "focus_areas" keys as they arrive. |
done | {job_id, status, charged_credits, output} | The final, authoritative result — read the review from output.output rather than trusting concatenated deltas, and the settled price from charged_credits. |
error | {code, message} | Replaces done when the run fails. |
# -N disables buffering so events print as they arrive
curl -N -s -X POST "$API/run-stream" \
-H "Authorization: Bearer $TOKEN" -H "Content-Type: application/json" \
-H "Idempotency-Key: cc-$(date +%s)" \
-d @input.json
# event: job
# data: {"job_id":"job_...","status":"running"}
#
# event: delta
# data: {"text":"{\"review_name\":\"acme-web"}
# ...
# event: done
# data: {"job_id":"job_...","status":"succeeded","charged_credits":612,"output":{"output":"{...}"}}
import json, requests
result = None
with requests.post(
API + "/run-stream",
headers={"Authorization": f"Bearer {TOKEN}",
"Idempotency-Key": "cc-001"},
json=payload,
stream=True,
) as r:
r.raise_for_status()
event = None
for line in r.iter_lines(decode_unicode=True):
if not line:
continue
if line.startswith("event:"):
event = line[len("event:"):].strip()
elif line.startswith("data:"):
data = json.loads(line[len("data:"):].strip())
if event == "delta":
print(".", end="", flush=True) # live progress
elif event == "done":
result = data
elif event == "error":
raise RuntimeError(data.get("message", "run failed"))
review = json.loads(result["output"]["output"]) # authoritative
print("charged:", result["charged_credits"], "-", review["review_name"])
print("posture:", review["posture"])
for f in review["findings"]:
print(f' [{f["priority"]}] {f["id"]} {f["resource"]}: {f["problem"]}')
with open("review.json", "w", encoding="utf-8") as fh:
json.dump(review, fh, indent=2)
if review["corrected_instructions"]:
with open("copilot-instructions.new.md", "w", encoding="utf-8") as fh:
fh.write(review["corrected_instructions"])
const res = await fetch(API + "/run-stream", {
method: "POST",
headers: {
Authorization: `Bearer ${TOKEN}`,
"Content-Type": "application/json",
"Idempotency-Key": crypto.randomUUID(),
},
body: JSON.stringify(payload),
});
const reader = res.body.getReader();
const decoder = new TextDecoder();
let buf = "", done = null;
for (;;) {
const chunk = await reader.read();
if (chunk.done) break;
buf += decoder.decode(chunk.value, { stream: true });
const frames = buf.split("\n\n");
buf = frames.pop();
for (const frame of frames) {
const name = /^event:\s*(.+)$/m.exec(frame)?.[1];
const body = /^data:\s*(.+)$/m.exec(frame)?.[1];
if (!name || !body) continue;
const data = JSON.parse(body);
if (name === "delta") process.stdout.write("."); // live progress
if (name === "done") done = data;
if (name === "error") throw new Error(data.message ?? "run failed");
}
}
const review = JSON.parse(done.output.output);
console.log(`\n${done.charged_credits} credits - ${review.review_name} [${review.posture}]`);
for (const f of review.findings) console.log(` [${f.priority}] ${f.id} ${f.resource}`);
writeFileSync("review.json", JSON.stringify(review, null, 2));
if (review.corrected_instructions) {
writeFileSync("copilot-instructions.new.md", review.corrected_instructions);
}
body, _ := json.Marshal(payload)
req, _ := http.NewRequest("POST", API+"/run-stream", bytes.NewReader(body))
req.Header.Set("Authorization", "Bearer "+token)
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Idempotency-Key", "cc-001")
res, err := http.DefaultClient.Do(req)
if err != nil {
log.Fatal(err)
}
defer res.Body.Close()
var event string
var final map[string]any
sc := bufio.NewScanner(res.Body)
sc.Buffer(make([]byte, 0, 64*1024), 4*1024*1024)
for sc.Scan() {
line := sc.Text()
switch {
case strings.HasPrefix(line, "event:"):
event = strings.TrimSpace(strings.TrimPrefix(line, "event:"))
case strings.HasPrefix(line, "data:"):
var data map[string]any
json.Unmarshal([]byte(strings.TrimPrefix(line, "data:")), &data)
switch event {
case "delta":
fmt.Print(".") // live progress
case "done":
final = data
case "error":
log.Fatal(data["message"])
}
}
}
// final["output"].(map[string]any)["output"].(string) is the review JSON - unmarshal it
// into the Review struct from step 4, write it to review.json, and write
// review.CorrectedInstructions to copilot-instructions.new.md when it is not empty.
// Java 17+ - read the stream line by line instead of buffering the body.
var req = HttpRequest.newBuilder(URI.create(API + "/run-stream"))
.header("Authorization", "Bearer " + TOKEN)
.header("Content-Type", "application/json")
.header("Idempotency-Key", "cc-001")
.POST(HttpRequest.BodyPublishers.ofString(jsonPayload))
.build();
var res = HTTP.send(req, HttpResponse.BodyHandlers.ofLines());
String event = null, done = null;
for (String line : (Iterable<String>) res.body()::iterator) {
if (line.startsWith("event:")) {
event = line.substring(6).trim();
} else if (line.startsWith("data:")) {
String data = line.substring(5).trim();
if ("delta".equals(event)) System.out.print("."); // live progress
else if ("done".equals(event)) done = data;
else if ("error".equals(event)) throw new RuntimeException(data);
}
}
// parse `done`, then parse data.output.output again - it is a JSON string holding
// review_name, posture, verdict, inventory[], findings[], coverage_check[],
// corrected_instructions, quick_wins[], focus_areas[] and the rest.
require "net/http"
require "json"
uri = URI(API + "/run-stream")
req = Net::HTTP::Post.new(uri)
req["Authorization"] = "Bearer #{TOKEN}"
req["Content-Type"] = "application/json"
req["Idempotency-Key"] = "cc-001"
req.body = payload.to_json
event = nil
done = nil
Net::HTTP.start(uri.host, uri.port, use_ssl: true) do |http|
http.request(req) do |res|
res.read_body do |chunk|
chunk.each_line do |line|
line = line.strip
if line.start_with?("event:")
event = line.delete_prefix("event:").strip
elsif line.start_with?("data:")
data = JSON.parse(line.delete_prefix("data:").strip)
case event
when "delta" then print "." # live progress
when "done" then done = data
when "error" then raise (data["message"] || "run failed")
end
end
end
end
end
end
review = JSON.parse(done["output"]["output"])
puts "\n#{done["charged_credits"]} credits - #{review["review_name"]} [#{review["posture"]}]"
review["findings"].each { |f| puts " [#{f["priority"]}] #{f["id"]} #{f["resource"]}" }
File.write("review.json", JSON.pretty_generate(review))
File.write("copilot-instructions.new.md", review["corrected_instructions"]) unless
review["corrected_instructions"].to_s.empty?
$event = null;
$done = null;
$ch = curl_init(API . "/run-stream");
curl_setopt_array($ch, [
CURLOPT_POST => true,
CURLOPT_HTTPHEADER => [
"Authorization: Bearer $TOKEN",
"Content-Type: application/json",
"Idempotency-Key: cc-001",
],
CURLOPT_POSTFIELDS => json_encode($payload),
CURLOPT_WRITEFUNCTION => function ($ch, $chunk) use (&$event, &$done) {
foreach (explode("\n", $chunk) as $line) {
$line = trim($line);
if (str_starts_with($line, "event:")) {
$event = trim(substr($line, 6));
} elseif (str_starts_with($line, "data:")) {
$data = json_decode(trim(substr($line, 5)), true);
if ($event === "delta") { echo "."; } // live progress
elseif ($event === "done") { $done = $data; }
elseif ($event === "error") { throw new Exception($data["message"] ?? "run failed"); }
}
}
return strlen($chunk);
},
]);
curl_exec($ch);
curl_close($ch);
$review = json_decode($done["output"]["output"], true);
echo "\n{$done['charged_credits']} credits - {$review['review_name']} [{$review['posture']}]\n";
foreach ($review["findings"] as $f) {
echo " [{$f['priority']}] {$f['id']} {$f['resource']}\n";
}
file_put_contents("review.json", json_encode($review, JSON_PRETTY_PRINT));
if (($review["corrected_instructions"] ?? "") !== "") {
file_put_contents("copilot-instructions.new.md", $review["corrected_instructions"]);
}
var req = new HttpRequestMessage(HttpMethod.Post, Api + "/run-stream") {
Content = JsonContent.Create(payload),
};
req.Headers.Add("Idempotency-Key", "cc-001");
using var res = await Http.SendAsync(req, HttpCompletionOption.ResponseHeadersRead);
using var reader = new StreamReader(await res.Content.ReadAsStreamAsync());
string? evt = null, done = null;
while (await reader.ReadLineAsync() is { } line)
{
if (line.StartsWith("event:")) evt = line[6..].Trim();
else if (line.StartsWith("data:"))
{
var data = line[5..].Trim();
if (evt == "delta") Console.Write("."); // live progress
else if (evt == "done") done = data;
else if (evt == "error") throw new Exception(data);
}
}
using var final = JsonDocument.Parse(done!);
var text = final.RootElement.GetProperty("output").GetProperty("output").GetString();
using var reviewDoc = JsonDocument.Parse(text!);
var review = reviewDoc.RootElement;
Console.WriteLine($"{review.GetProperty("review_name")} [{review.GetProperty("posture")}]");
foreach (var f in review.GetProperty("findings").EnumerateArray())
Console.WriteLine($" [{f.GetProperty("priority")}] {f.GetProperty("id")} {f.GetProperty("resource")}");
await File.WriteAllTextAsync("review.json", text!);
var rewritten = review.GetProperty("corrected_instructions").GetString();
if (!string.IsNullOrEmpty(rewritten))
await File.WriteAllTextAsync("copilot-instructions.new.md", rewritten);
In a browser, the native EventSource only speaks GET, and this endpoint is a
POST — read the fetch response body incrementally, as the JavaScript
sample above does. On an idempotent replay the server may answer with a plain JSON
envelope instead of an event stream; check the Content-Type before you start
parsing frames.