Locate the first usable procedure in a ranked list

Define the local evaluation task

For a synthetic procedure search, each ordered result list has explicitly labelled usable document IDs. At most ten queries and ten hits per query are accepted. Define a local score as the average of 1/r, where r is the first usable position starting at one; a list with no usable hit contributes zero. Keep the fraction exact. This is a supplied scoring rule, not a measured search result or a release threshold.

Locate first usable position per query
Use zero for a supplied miss
Average exact inverse positions

An unlabelled usable set must be prepared before scoring.

Use the filled cases

Original synthetic input and literal answer key
Case	Supplied input	Expected structured output	Decision and reason
Ranks two and four	{"queries":[{"hits":["X","D"],"usable":["D"]},{"hits":["A","B","C","E"],"usable":["E"]}]}	{"ranks":[2,4],"numerator":3,"denominator":8}	(1/2+1/4)/2 is 3/8.
One query has no useful hit	{"queries":[{"hits":["D"],"usable":["D"]},{"hits":["X"],"usable":["E"]}]}	{"ranks":[1,null],"numerator":1,"denominator":2}	The miss contributes zero while remaining in the two-query average.
Usable labels absent despite empty hits	{"queries":[{"hits":[]}]}	{"error":"usable-labels"}	Empty hits cannot turn an absent usable-label channel into a known zero score.
No query observations	{"queries":[]}	{"ranks":[],"numerator":0,"denominator":0}	No average is established without observations; do not report a perfect score.

Download editable CSV ↓

Work the no-queries case. Which output is required, and why?

Show the worked answer

{"ranks":[],"numerator":0,"denominator":0} — No average is established without observations; do not report a perfect score.

Run the self-contained evaluator

Save as .mjs and run with Node.js; supplied fixtures only

import assert from 'node:assert/strict';

function evaluate(x){
 if(!Array.isArray(x.queries)||x.queries.some(q=>!q||!Array.isArray(q.hits)||!Array.isArray(q.usable)))return {error:'usable-labels'};
 if(x.queries.length>10||x.queries.some(q=>q.hits.length>10))return {error:'score-bounds'};
 const gcd=(a,b)=>b?gcd(b,a%b):a;let n=0,d=1;
 const ranks=x.queries.map(q=>{const i=q.hits.findIndex(id=>q.usable.includes(id));return i<0?null:i+1;});
 for(const r of ranks)if(r){n=n*r+d;d*=r;const g=gcd(n,d);n/=g;d/=g;}
 if(!ranks.length)return {ranks,numerator:0,denominator:0};
 d*=ranks.length;const g=gcd(n,d);return {ranks,numerator:n/g,denominator:d/g};
}

const cases = [
  {
    "id": "two-ranks",
    "input": {
      "queries": [
        {
          "hits": [
            "X",
            "D"
          ],
          "usable": [
            "D"
          ]
        },
        {
          "hits": [
            "A",
            "B",
            "C",
            "E"
          ],
          "usable": [
            "E"
          ]
        }
      ]
    },
    "expected": {
      "ranks": [
        2,
        4
      ],
      "numerator": 3,
      "denominator": 8
    }
  },
  {
    "id": "one-miss",
    "input": {
      "queries": [
        {
          "hits": [
            "D"
          ],
          "usable": [
            "D"
          ]
        },
        {
          "hits": [
            "X"
          ],
          "usable": [
            "E"
          ]
        }
      ]
    },
    "expected": {
      "ranks": [
        1,
        null
      ],
      "numerator": 1,
      "denominator": 2
    }
  },
  {
    "id": "missing-usable-channel",
    "input": {
      "queries": [
        {
          "hits": []
        }
      ]
    },
    "expected": {
      "error": "usable-labels"
    }
  },
  {
    "id": "no-queries",
    "input": {
      "queries": []
    },
    "expected": {
      "ranks": [],
      "numerator": 0,
      "denominator": 0
    }
  }
];
for (const c of cases) assert.deepEqual(evaluate(c.input), c.expected, c.id);
console.log(JSON.stringify({checked:cases.length,caseIds:cases.map(c=>c.id)}));

Download runnable example ↓

Retain the result order, usable labels and score definition together. A high position score does not measure answer correctness, document freshness or staff task completion.

Define the local evaluation task

Use the filled cases

Run the self-contained evaluator

Related resources