updates

modules/capabilities/_diarize-transcript.py

@@ -2,6 +2,7 @@ import argparse
 import json
 import re
 from dataclasses import dataclass
+from itertools import groupby
 from pathlib import Path
 
 
@@ -25,9 +26,13 @@ def load_tokens(path: Path) -> list[Token]:
 
     tokens: list[Token] = []
     pending = ""
+    # Whisper reserves all IDs from end-of-text onward for control tokens.
+    end_of_text = 50256 + int(transcript["model"]["multilingual"])
 
     for segment in transcript["transcription"]:
         for raw_token in segment["tokens"]:
+            if raw_token["id"] >= end_of_text:
+                continue
             text = raw_token["text"]
             offsets = raw_token.get("offsets")
 
@@ -81,19 +86,12 @@ def token_speaker(token: Token, turns: list[Turn]) -> str:
 
 def render(tokens: list[Token], turns: list[Turn]) -> str:
     speaker_numbers: dict[str, int] = {}
-    blocks: list[tuple[int, str]] = []
-
-    for token in tokens:
-        speaker = token_speaker(token, turns)
-        number = speaker_numbers.setdefault(speaker, len(speaker_numbers) + 1)
-
-        if blocks and blocks[-1][0] == number:
-            blocks[-1] = number, blocks[-1][1] + token.text
-        else:
-            blocks.append((number, token.text))
-
     rendered = []
-    for number, text in blocks:
+    for speaker, group in groupby(
+        tokens, key=lambda token: token_speaker(token, turns)
+    ):
+        number = speaker_numbers.setdefault(speaker, len(speaker_numbers) + 1)
+        text = "".join(token.text for token in group)
         text = re.sub(r"\s+", " ", text).strip()
         if text:
             rendered.append(f"**Speaker {number}:**\n\n{text}")

modules/capabilities/_transcribe-recordings.sh

@@ -17,48 +17,47 @@ move_no_clobber() {
     return 1
   fi
 
-  mv -T --no-clobber -- "$source" "$destination"
+  mv -T --no-clobber -- "$source" "$destination" || return 1
   ! path_exists "$source" && path_exists "$destination"
 }
 
-fail_recording() {
+fail_recording() (
   local source=$1
   local failed_dir=$2
   local message=$3
   local details=${4:-}
   local base target error_tmp suffix=0
 
-  base=$(basename "$source")
+  base=${source##*/}
   target="$failed_dir/$base"
-  rm -f -- "${source}.transcribe-state"
 
   while path_exists "$target" || path_exists "${target}.error.txt"; do
     ((suffix += 1))
     target="$failed_dir/$base.failed-$suffix"
   done
 
+  error_tmp=$(mktemp "$failed_dir/.transcription-error.XXXXXX") || return 1
+  trap 'rm -f -- "$error_tmp"' EXIT
+  {
+    printf '%s\n' "$message" || return 1
+    if [[ -n "$details" && -s "$details" ]]; then
+      printf '\n' || return 1
+      cat -- "$details" || return 1
+    fi
+  } >"$error_tmp" || return 1
+
   if ! move_no_clobber "$source" "$target"; then
     printf 'Could not quarantine %s: destination collision or move failure\n' "$source" >&2
     return 1
   fi
 
-  error_tmp=$(mktemp "$failed_dir/.transcription-error.XXXXXX")
-  {
-    printf '%s\n' "$message"
-    if [[ -n "$details" && -s "$details" ]]; then
-      printf '\n'
-      cat -- "$details"
-    fi
-  } >"$error_tmp"
-
   if ! ln -- "$error_tmp" "${target}.error.txt"; then
     printf 'Could not publish error sidecar for %s\n' "$target" >&2
-    rm -f -- "$error_tmp"
     return 1
   fi
-  rm -f -- "$error_tmp"
+  rm -f -- "${source}.transcribe-state" || return 1
   printf 'Moved failed recording to %s\n' "$target" >&2
-}
+)
 
 recover_completed_recording() {
   local source=$1
@@ -71,15 +70,18 @@ recover_completed_recording() {
   fi
 
   if ! path_exists "$transcript"; then
-    rm -f -- "$state_file"
+    rm -f -- "$state_file" || return 3
     return 1
   fi
 
   if ! move_no_clobber "$source" "$archive"; then
-    return 2
+    if path_exists "$archive"; then
+      return 2
+    fi
+    return 3
   fi
 
-  rm -f -- "$state_file"
+  rm -f -- "$state_file" || return 3
   printf 'Recovered completed transcription and archived %s\n' "$archive"
   return 0
 }
@@ -93,10 +95,11 @@ process_recording() (
   local archive_dir=$4
   local failed_dir=$5
   local base stem transcript archive original_path recorded_at transcribed_at duration
-  local work_dir log_file raw_output rendered_output draft state_file recovery_status
+  local work_dir log_file raw_output rendered_output state_file recovery_status
+  local draft="" diarization_log
   local diarization_succeeded=false
 
-  base=$(basename "$source")
+  base=${source##*/}
   stem=${base%.*}
   transcript="$transcript_dir/$stem.md"
   archive="$archive_dir/$base"
@@ -112,6 +115,10 @@ process_recording() (
     fail_recording "$source" "$failed_dir" "Archive collision after transcription: $archive"
     return 1
   fi
+  if [[ $recovery_status -ne 1 ]]; then
+    printf 'Could not recover %s; leaving state for retry\n' "$source" >&2
+    return 1
+  fi
 
   if path_exists "$transcript"; then
     fail_recording "$source" "$failed_dir" "Transcript already exists: $transcript"
@@ -122,11 +129,13 @@ process_recording() (
     return 1
   fi
 
-  recorded_at=$(date --date="@$(stat -c %Y -- "$source")" --iso-8601=seconds)
+  recorded_at=$(stat -c %Y -- "$source")
+  recorded_at=$(date --date="@$recorded_at" --iso-8601=seconds)
   transcribed_at=$(date --utc --iso-8601=seconds)
   work_dir=$(mktemp -d "$runtime_dir/work.XXXXXX")
-  trap 'rm -r -- "$work_dir"' EXIT
+  trap 'if [[ -n "$draft" ]]; then rm -f -- "$draft"; fi; rm -r -- "$work_dir"' EXIT
   log_file="$work_dir/transcription.log"
+  diarization_log="$work_dir/diarization.log"
   raw_output="$work_dir/transcript"
   rendered_output="$work_dir/transcript.md"
 
@@ -175,11 +184,14 @@ process_recording() (
       --audio "$work_dir/audio.wav" \
       --model "$TRANSCRIBE_DIARIZATION_MODEL" \
       --transcript "${raw_output}.json" \
-      --output "$rendered_output" >>"$log_file" 2>&1 \
+      --output "$rendered_output" >"$diarization_log" 2>&1 \
     && [[ -s "$rendered_output" ]]; then
     diarization_succeeded=true
   else
     printf 'Speaker diarization unavailable; publishing unlabeled transcript\n' >&2
+    if [[ -s "$diarization_log" ]]; then
+      cat -- "$diarization_log" >&2
+    fi
     cp -- "${raw_output}.txt" "$rendered_output"
   fi
 
@@ -222,54 +234,70 @@ process_recording() (
   printf 'Transcribed %s to %s\n' "$archive" "$transcript"
 )
 
-process_scope() {
+process_scope() (
   local recordings_dir=$1
   local transcript_dir=$2
   local inbox_dir="$recordings_dir/inbox"
   local processing_dir="$recordings_dir/processing"
   local archive_dir="$recordings_dir/archive"
   local failed_dir="$recordings_dir/failed"
-  local source processing_target recording_status signature index scope_status=0
+  local source processing_target recording_status signature index listing scope_status=0
   local -a sources signatures
 
   mkdir -p "$inbox_dir" "$processing_dir" "$archive_dir" "$failed_dir" "$transcript_dir" || return 1
+  listing=$(mktemp "$runtime_dir/recordings.XXXXXX") || return 1
+  trap 'rm -f -- "$listing"' EXIT
 
+  find "$processing_dir" -maxdepth 1 -type f -print0 | sort -z >"$listing" || return 1
   while IFS= read -r -d '' source; do
     if is_recording "$source"; then
+      # Keep this call outside conditionals so its subshell's errexit stays active.
       process_recording "$source" "$inbox_dir" "$transcript_dir" "$archive_dir" "$failed_dir"
       recording_status=$?
       if [[ $recording_status -ne 0 ]]; then
         scope_status=1
       fi
     fi
-  done < <(find "$processing_dir" -maxdepth 1 -type f -print0 | sort -z)
+  done <"$listing"
 
   while true; do
     sources=()
     signatures=()
 
+    find "$inbox_dir" -maxdepth 1 -type f -print0 | sort -z >"$listing" || return 1
     while IFS= read -r -d '' source; do
-      if is_recording "$source" && signature=$(stat -c '%s:%y:%i' -- "$source"); then
+      if is_recording "$source"; then
+        if ! signature=$(stat -c '%s:%y:%i' -- "$source"); then
+          if path_exists "$source"; then return 1; fi
+          continue
+        fi
         sources+=("$source")
         signatures+=("$signature")
       fi
-    done < <(find "$inbox_dir" -maxdepth 1 -type f -print0 | sort -z)
+    done <"$listing"
 
     if [[ ${#sources[@]} -eq 0 ]]; then
       break
     fi
 
-    sleep "${TRANSCRIBE_SETTLE_SECONDS:-300}"
+    sleep "${TRANSCRIBE_SETTLE_SECONDS:-300}" || return 1
 
     for index in "${!sources[@]}"; do
       source=${sources[$index]}
-      if [[ ! -f "$source" ]] || [[ $(stat -c '%s:%y:%i' -- "$source") != "${signatures[$index]}" ]]; then
+      if [[ ! -f "$source" ]]; then
+        continue
+      fi
+      if ! signature=$(stat -c '%s:%y:%i' -- "$source"); then
+        if path_exists "$source"; then return 1; fi
+        continue
+      fi
+      if [[ $signature != "${signatures[$index]}" ]]; then
         continue
       fi
 
-      processing_target="$processing_dir/$(basename "$source")"
+      processing_target="$processing_dir/${source##*/}"
       if ! move_no_clobber "$source" "$processing_target"; then
-        fail_recording "$source" "$failed_dir" "Processing filename already exists: $processing_target"
+        fail_recording "$source" "$failed_dir" "Could not claim recording: destination collision or move failure: $processing_target" || return 1
         scope_status=1
         continue
       fi
@@ -282,7 +310,7 @@ process_scope() {
   done
 
   return "$scope_status"
-}
+)
 
 runtime_dir="${XDG_RUNTIME_DIR:?}/transcribe-recordings"
 mkdir -p "$runtime_dir"

modules/capabilities/dev-tools.nix

@@ -38,6 +38,7 @@
         kubelogin-oidc
         rclone
         scc
+        visidata
         just
         gardenctl
         gardenlogin