updates

modules/capabilities/transcription/transcribe-recordings.sh

@@ -0,0 +1,319 @@
+path_exists() {
+  [[ -e "$1" || -L "$1" ]]
+}
+
+is_recording() {
+  case "${1,,}" in
+    *.wav | *.flac | *.mp3 | *.ogg | *.opus | *.aac | *.m4a) return 0 ;;
+    *) return 1 ;;
+  esac
+}
+
+move_no_clobber() {
+  local source=$1
+  local destination=$2
+
+  if path_exists "$destination"; then
+    return 1
+  fi
+
+  mv -T --no-clobber -- "$source" "$destination" || return 1
+  ! path_exists "$source" && path_exists "$destination"
+}
+
+fail_recording() (
+  local source=$1
+  local failed_dir=$2
+  local message=$3
+  local details=${4:-}
+  local base target error_tmp suffix=0
+
+  base=${source##*/}
+  target="$failed_dir/$base"
+
+  while path_exists "$target" || path_exists "${target}.error.txt"; do
+    ((suffix += 1))
+    target="$failed_dir/$base.failed-$suffix"
+  done
+
+  error_tmp=$(mktemp "$failed_dir/.transcription-error.XXXXXX") || return 1
+  trap 'rm -f -- "$error_tmp"' EXIT
+  {
+    printf '%s\n' "$message" || return 1
+    if [[ -n "$details" && -s "$details" ]]; then
+      printf '\n' || return 1
+      cat -- "$details" || return 1
+    fi
+  } >"$error_tmp" || return 1
+
+  if ! move_no_clobber "$source" "$target"; then
+    printf 'Could not quarantine %s: destination collision or move failure\n' "$source" >&2
+    return 1
+  fi
+
+  if ! ln -- "$error_tmp" "${target}.error.txt"; then
+    printf 'Could not publish error sidecar for %s\n' "$target" >&2
+    return 1
+  fi
+  rm -f -- "${source}.transcribe-state" || return 1
+  printf 'Moved failed recording to %s\n' "$target" >&2
+)
+
+recover_completed_recording() {
+  local source=$1
+  local transcript=$2
+  local archive=$3
+  local state_file="${source}.transcribe-state"
+
+  if [[ ! -f "$state_file" ]]; then
+    return 1
+  fi
+
+  if ! path_exists "$transcript"; then
+    rm -f -- "$state_file" || return 3
+    return 1
+  fi
+
+  if ! move_no_clobber "$source" "$archive"; then
+    if path_exists "$archive"; then
+      return 2
+    fi
+    return 3
+  fi
+
+  rm -f -- "$state_file" || return 3
+  printf 'Recovered completed transcription and archived %s\n' "$archive"
+  return 0
+}
+
+process_recording() (
+  set -o errexit
+
+  local source=$1
+  local inbox_dir=$2
+  local transcript_dir=$3
+  local archive_dir=$4
+  local failed_dir=$5
+  local base stem transcript archive original_path recorded_at transcribed_at duration
+  local work_dir log_file raw_output rendered_output state_file recovery_status
+  local draft="" diarization_log
+  local diarization_succeeded=false
+
+  base=${source##*/}
+  stem=${base%.*}
+  transcript="$transcript_dir/$stem.md"
+  archive="$archive_dir/$base"
+  original_path="$inbox_dir/$base"
+  state_file="${source}.transcribe-state"
+
+  if recover_completed_recording "$source" "$transcript" "$archive"; then
+    return 0
+  else
+    recovery_status=$?
+  fi
+  if [[ $recovery_status -eq 2 ]]; then
+    fail_recording "$source" "$failed_dir" "Archive collision after transcription: $archive"
+    return 1
+  fi
+  if [[ $recovery_status -ne 1 ]]; then
+    printf 'Could not recover %s; leaving state for retry\n' "$source" >&2
+    return 1
+  fi
+
+  if path_exists "$transcript"; then
+    fail_recording "$source" "$failed_dir" "Transcript already exists: $transcript"
+    return 1
+  fi
+  if path_exists "$archive"; then
+    fail_recording "$source" "$failed_dir" "Archived recording already exists: $archive"
+    return 1
+  fi
+
+  recorded_at=$(stat -c %Y -- "$source")
+  recorded_at=$(date --date="@$recorded_at" --iso-8601=seconds)
+  transcribed_at=$(date --utc --iso-8601=seconds)
+  work_dir=$(mktemp -d "$runtime_dir/work.XXXXXX")
+  trap 'if [[ -n "$draft" ]]; then rm -f -- "$draft"; fi; rm -r -- "$work_dir"' EXIT
+  log_file="$work_dir/transcription.log"
+  diarization_log="$work_dir/diarization.log"
+  raw_output="$work_dir/transcript"
+  rendered_output="$work_dir/transcript.md"
+
+  if ! duration=$(ffprobe \
+    -v error \
+    -show_entries format=duration \
+    -of default=noprint_wrappers=1:nokey=1 \
+    "$source" 2>"$log_file"); then
+    fail_recording "$source" "$failed_dir" "Media validation failed" "$log_file"
+    return 1
+  fi
+  if [[ ! $duration =~ ^[0-9]+([.][0-9]+)?$ || -z ${duration//[.0]/} ]]; then
+    printf 'Invalid media duration: %s\n' "$duration" >"$log_file"
+    fail_recording "$source" "$failed_dir" "Media validation failed" "$log_file"
+    return 1
+  fi
+
+  if ! ffmpeg \
+    -nostdin \
+    -hide_banner \
+    -loglevel error \
+    -i "$source" \
+    -vn \
+    -ac 1 \
+    -ar 16000 \
+    -c:a pcm_s16le \
+    "$work_dir/audio.wav" >"$log_file" 2>&1; then
+    fail_recording "$source" "$failed_dir" "FFmpeg conversion failed" "$log_file"
+    return 1
+  fi
+
+  if ! whisper-cli \
+    --model "$TRANSCRIBE_MODEL" \
+    --file "$work_dir/audio.wav" \
+    --language auto \
+    --output-json-full \
+    --output-txt \
+    --output-file "$raw_output" >>"$log_file" 2>&1 \
+    || [[ ! -s "${raw_output}.json" || ! -s "${raw_output}.txt" ]]; then
+    fail_recording "$source" "$failed_dir" "Whisper transcription failed" "$log_file"
+    return 1
+  fi
+
+  if [[ -f "$TRANSCRIBE_DIARIZATION_MODEL/.complete" && -f "$TRANSCRIBE_DIARIZATION_MODEL/config.yaml" ]] \
+    && diarize-transcript \
+      --audio "$work_dir/audio.wav" \
+      --model "$TRANSCRIBE_DIARIZATION_MODEL" \
+      --transcript "${raw_output}.json" \
+      --output "$rendered_output" >"$diarization_log" 2>&1 \
+    && [[ -s "$rendered_output" ]]; then
+    diarization_succeeded=true
+  else
+    printf 'Speaker diarization unavailable; publishing unlabeled transcript\n' >&2
+    if [[ -s "$diarization_log" ]]; then
+      cat -- "$diarization_log" >&2
+    fi
+    cp -- "${raw_output}.txt" "$rendered_output"
+  fi
+
+  draft=$(mktemp "$transcript_dir/.transcript.XXXXXX")
+  {
+    printf '# %s\n\n' "$stem"
+    printf -- '- Original recording: %s\n' "$original_path"
+    printf -- '- Recording timestamp: %s\n' "$recorded_at"
+    printf -- '- Transcription timestamp: %s\n' "$transcribed_at"
+    printf -- '- Duration: %s seconds\n' "$duration"
+    printf -- '- Engine: whisper.cpp %s (%s)\n' "$TRANSCRIBE_ENGINE_VERSION" "$TRANSCRIBE_ENGINE_ACCELERATION"
+    printf -- '- Model: %s\n' "$TRANSCRIBE_MODEL_NAME"
+    printf -- '- Model SHA-256: %s\n' "$TRANSCRIBE_MODEL_SHA256"
+    if [[ $diarization_succeeded == true ]]; then
+      printf -- '- Diarization engine: pyannote.audio %s (CPU)\n' "$TRANSCRIBE_DIARIZATION_ENGINE_VERSION"
+      printf -- '- Diarization model: %s\n' "$TRANSCRIBE_DIARIZATION_MODEL_NAME"
+      printf -- '- Diarization model revision: %s\n\n' "$TRANSCRIBE_DIARIZATION_MODEL_REVISION"
+    else
+      printf -- '- Diarization: unavailable; unlabeled fallback published\n\n'
+    fi
+    cat -- "$rendered_output"
+  } >"$draft"
+  chmod 0644 "$draft"
+
+  touch -- "$state_file"
+
+  if ! ln -- "$draft" "$transcript"; then
+    rm -f -- "$draft" "$state_file"
+    fail_recording "$source" "$failed_dir" "Transcript appeared while transcription was running: $transcript" "$log_file"
+    return 1
+  fi
+  rm -f -- "$draft"
+
+  if ! move_no_clobber "$source" "$archive"; then
+    printf 'Could not archive %s; leaving recoverable state in processing\n' "$source" >&2
+    return 1
+  fi
+
+  rm -f -- "$state_file"
+  printf 'Transcribed %s to %s\n' "$archive" "$transcript"
+)
+
+process_scope() (
+  local recordings_dir=$1
+  local transcript_dir=$2
+  local inbox_dir="$recordings_dir/inbox"
+  local processing_dir="$recordings_dir/processing"
+  local archive_dir="$recordings_dir/archive"
+  local failed_dir="$recordings_dir/failed"
+  local source processing_target recording_status signature index listing scope_status=0
+  local -a sources signatures
+
+  mkdir -p "$inbox_dir" "$processing_dir" "$archive_dir" "$failed_dir" "$transcript_dir" || return 1
+  listing=$(mktemp "$runtime_dir/recordings.XXXXXX") || return 1
+  trap 'rm -f -- "$listing"' EXIT
+
+  find "$processing_dir" -maxdepth 1 -type f -print0 | sort -z >"$listing" || return 1
+  while IFS= read -r -d '' source; do
+    if is_recording "$source"; then
+      # Keep this call outside conditionals so its subshell's errexit stays active.
+      process_recording "$source" "$inbox_dir" "$transcript_dir" "$archive_dir" "$failed_dir"
+      recording_status=$?
+      if [[ $recording_status -ne 0 ]]; then
+        scope_status=1
+      fi
+    fi
+  done <"$listing"
+
+  while true; do
+    sources=()
+    signatures=()
+
+    find "$inbox_dir" -maxdepth 1 -type f -print0 | sort -z >"$listing" || return 1
+    while IFS= read -r -d '' source; do
+      if is_recording "$source"; then
+        if ! signature=$(stat -c '%s:%y:%i' -- "$source"); then
+          if path_exists "$source"; then return 1; fi
+          continue
+        fi
+        sources+=("$source")
+        signatures+=("$signature")
+      fi
+    done <"$listing"
+
+    if [[ ${#sources[@]} -eq 0 ]]; then
+      break
+    fi
+
+    sleep "${TRANSCRIBE_SETTLE_SECONDS:-300}" || return 1
+
+    for index in "${!sources[@]}"; do
+      source=${sources[$index]}
+      if [[ ! -f "$source" ]]; then
+        continue
+      fi
+      if ! signature=$(stat -c '%s:%y:%i' -- "$source"); then
+        if path_exists "$source"; then return 1; fi
+        continue
+      fi
+      if [[ $signature != "${signatures[$index]}" ]]; then
+        continue
+      fi
+
+      processing_target="$processing_dir/${source##*/}"
+      if ! move_no_clobber "$source" "$processing_target"; then
+        fail_recording "$source" "$failed_dir" "Could not claim recording: destination collision or move failure: $processing_target" || return 1
+        scope_status=1
+        continue
+      fi
+      process_recording "$processing_target" "$inbox_dir" "$transcript_dir" "$archive_dir" "$failed_dir"
+      recording_status=$?
+      if [[ $recording_status -ne 0 ]]; then
+        scope_status=1
+      fi
+    done
+  done
+
+  return "$scope_status"
+)
+
+runtime_dir="${XDG_RUNTIME_DIR:?}/transcribe-recordings"
+mkdir -p "$runtime_dir"
+
+set +o errexit
+process_scope "$TRANSCRIBE_SYNC_ROOT/recordings" "$TRANSCRIBE_SYNC_ROOT/notes/transcripts"