Monitor quality in production

import { GradeLevelAppropriatenessEvaluator } from "@learning-commons/evaluators";

const sample = [
  {
    id: "passage-1",
    text: "Bees visit flowers to collect nectar. They carry it back to the hive and turn it into honey.",
  },
  {
    id: "passage-2",
    text: "Honeybees forage for nectar and convert it into honey through enzymatic processing inside the colony.",
  },
  {
    id: "passage-3",
    text: "A hive is a home for bees. The bees make honey and store it there.",
  },
];

const gradeLevel = new GradeLevelAppropriatenessEvaluator({
  googleApiKey: process.env.GOOGLE_API_KEY,
});

const scored = [];

for (const item of sample) {
  const { result } = await gradeLevel.evaluate({ text: item.text });
  scored.push({ id: item.id, gradeBand: result.grade_band });
  console.log(item.id, result.grade_band); // "K-1" | "2-3" | "4-5" | "6-8" | "9-10" | "11-12"
}

import os

from learning_commons_evaluators import GradeLevelAppropriatenessEvaluator

sample = [
  {
    "id": "passage-1",
    "text": "Bees visit flowers to collect nectar. They carry it back to the hive and turn it into honey.",
  },
  {
    "id": "passage-2",
    "text": "Honeybees forage for nectar and convert it into honey through enzymatic processing inside the colony.",
  },
  {
    "id": "passage-3",
    "text": "A hive is a home for bees. The bees make honey and store it there.",
  },
]

grade_level = GradeLevelAppropriatenessEvaluator(
  google_api_key=os.environ["GOOGLE_API_KEY"],
)

scored = []

for item in sample:
  result = grade_level.evaluate_sync(text=item["text"]).result
  scored.append({"id": item["id"], "grade_band": result.grade_band})
  print(item["id"], result.grade_band)  # "K-1" | "2-3" | "4-5" | "6-8" | "9-10" | "11-12"

const GRADE_BANDS = ["K-1", "2-3", "4-5", "6-8", "9-10", "11-12"] as const;
const TARGET_BAND = "4-5";

// Count saved from the previous run. Replace this with your baseline.
const baselineAboveTarget = 0;

const currentAboveTarget = scored.filter(
  (item) => GRADE_BANDS.indexOf(item.gradeBand) > GRADE_BANDS.indexOf(TARGET_BAND),
).length;

console.log(`${currentAboveTarget} of ${scored.length} passages are above ${TARGET_BAND}.`);

if (currentAboveTarget > baselineAboveTarget) {
  console.log("Drift flagged. Investigate before this sample's pattern reaches students.");
} else {
  console.log("No increase above the target band.");
}

GRADE_BANDS = ["K-1", "2-3", "4-5", "6-8", "9-10", "11-12"]
TARGET_BAND = "4-5"

# Count saved from the previous run. Replace this with your baseline.
baseline_above_target = 0

current_above_target = sum(
  1
  for item in scored
  if GRADE_BANDS.index(item["grade_band"]) > GRADE_BANDS.index(TARGET_BAND)
)

print(f"{current_above_target} of {len(scored)} passages are above {TARGET_BAND}.")

if current_above_target > baseline_above_target:
  print("Drift flagged. Investigate before this sample's pattern reaches students.")
else:
  print("No increase above the target band.")

import { VocabularyComplexityEvaluator } from "@learning-commons/evaluators";

const HARD_SCORES = ["very_complex", "exceedingly_complex"];

const vocabulary = new VocabularyComplexityEvaluator({
  googleApiKey: process.env.GOOGLE_API_KEY,
  openaiApiKey: process.env.OPENAI_API_KEY,
});

let hardVocabulary = 0;

for (const item of sample) {
  const { result } = await vocabulary.evaluate({
    text: item.text,
    grade_level: "4",
  });

  console.log(item.id, result.complexity_score);
  console.log(result.tier_3_words);

  if (HARD_SCORES.includes(result.complexity_score)) {
    hardVocabulary += 1;
  }
}

console.log(`${hardVocabulary} of ${sample.length} passages have very or exceedingly complex vocabulary.`);

from learning_commons_evaluators import VocabularyComplexityEvaluator

HARD_SCORES = {"very_complex", "exceedingly_complex"}

vocabulary = VocabularyComplexityEvaluator(
  google_api_key=os.environ["GOOGLE_API_KEY"],
  openai_api_key=os.environ["OPENAI_API_KEY"],
)

hard_vocabulary = 0

for item in sample:
  result = vocabulary.evaluate_sync(text=item["text"], grade_level=4).result

  print(item["id"], result.complexity_score)
  print(result.tier_3_words)

  if result.complexity_score in HARD_SCORES:
    hard_vocabulary += 1

print(f"{hard_vocabulary} of {len(sample)} passages have very or exceedingly complex vocabulary.")