;;; yeetube-scraper.el --- Pure YouTube JSON parsing for yeetube  -*- lexical-binding: t; -*-

;; Copyright (C) 2023-2024  Thanos Apollo

;; Author: Thanos Apollo <public@thanosapollo.org>
;; Keywords: extensions youtube videos
;; URL: https://thanosapollo.org/projects/yeetube/

;; This program is free software; you can redistribute it and/or modify
;; it under the terms of the GNU General Public License as published by
;; the Free Software Foundation, either version 3 of the License, or
;; (at your option) any later version.

;;; Commentary:

;; Pure data extraction from YouTube's ytInitialData JSON.
;; All private functions are side-effect free.
;; The public entry point `yeetube-scraper-parse' reads from the
;; current buffer but restores point via `save-excursion'.
;; Input: parsed JSON (alists).  Output: plists.

;;; Code:

(require 'cl-lib)
(require 'subr-x)

(defun yeetube-scraper--text-from-object (obj)
  "Return text from OBJ, a \"simpleText\" string or segment-array wrapper.
YouTube returns text fields in two forms depending on layout: a
single \"simpleText\" string, or a \"runs\" array of segments to be
concatenated.  Returns nil when neither shape is present."
  (cond ((null obj) nil)
        ((alist-get 'simpleText obj))
        ((alist-get 'runs obj)
         (mapconcat (lambda (run) (or (alist-get 'text run) ""))
                    (alist-get 'runs obj)))))

(defun yeetube-scraper--byline-runs (renderer)
  "Return RENDERER's first non-empty byline segment list.
Each candidate holds a \"runs\" array; mirrors the fallback chain
YouTube uses across layouts: \"longBylineText\" (search/channel
pages), \"ownerText\" (some playlist views), \"shortBylineText\"
\(compact contexts)."
  (cl-some (lambda (key)
             (alist-get 'runs (alist-get key renderer)))
           '(longBylineText ownerText shortBylineText)))

(defun yeetube-scraper--extract-video (renderer)
  "Extract a video plist from a videoRenderer RENDERER alist."
  (let* ((title (yeetube-scraper--text-from-object
                 (alist-get 'title renderer)))
         (views (yeetube-scraper--text-from-object
                 (alist-get 'viewCountText renderer)))
         (duration (yeetube-scraper--text-from-object
                    (alist-get 'lengthText renderer)))
         (date-raw (yeetube-scraper--text-from-object
                    (alist-get 'publishedTimeText renderer)))
         (date (when date-raw (string-replace "Streamed " "" date-raw)))
         (byline-runs (yeetube-scraper--byline-runs renderer))
         (channel (alist-get 'text (car byline-runs)))
         (browse-endpoint (alist-get 'browseEndpoint
				     (alist-get 'navigationEndpoint (car byline-runs))))
         (channel-id (alist-get 'canonicalBaseUrl browse-endpoint))
         (browse-id (alist-get 'browseId browse-endpoint))
         (thumbs (alist-get 'thumbnails (alist-get 'thumbnail renderer)))
         (thumb-url (yeetube-scraper--thumbnail-url
                     (alist-get 'videoId renderer) thumbs)))
    (list :id (alist-get 'videoId renderer)
          :title (or title "")
          :views (or views "")
          :duration (or duration "")
          :date (or date "")
          :channel (or channel "")
          :channel-id (or channel-id "")
          :browse-id (or browse-id "")
          :thumbnail-url (or thumb-url "")
          :type 'video)))

(defun yeetube-scraper--thumbnail-url (video-id thumbnails)
  "Build a small thumbnail URL for VIDEO-ID from THUMBNAILS list.
Falls back to the predictable default.jpg URL pattern.
Normalises any hq720 / hqdefault / mqdefault / sddefault /
maxresdefault variant to default.jpg so the tabulated UI fetches
the smallest image."
  (or (and-let* ((url (alist-get 'url (car thumbnails)))
                 (qmark (string-search "?" url))
                 (base (substring url 0 qmark)))
        (replace-regexp-in-string
         "/\\(hq720\\|hqdefault\\|mqdefault\\|sddefault\\|maxresdefault\\)\\.jpg\\'"
         "/default.jpg" base))
      (format "https://i.ytimg.com/vi/%s/default.jpg" video-id)))

;;; lockupViewModel helpers (shared by video and playlist extraction)

(defun yeetube-scraper--lockup-metadata-parts (renderer)
  "Return metadataParts from all rows of a lockupViewModel RENDERER."
  (let* ((meta (alist-get 'lockupMetadataViewModel
                          (alist-get 'metadata renderer)))
         (cmvm (alist-get 'contentMetadataViewModel
                          (alist-get 'metadata meta)))
         (rows (alist-get 'metadataRows cmvm)))
    (cl-loop for row in rows
             append (alist-get 'metadataParts row))))

(defun yeetube-scraper--lockup-part-text (part)
  "Return the text content from a single metadataParts PART, or nil."
  (alist-get 'content (alist-get 'text part)))

(defun yeetube-scraper--lockup-views-and-date (parts)
  "Return cons (VIEWS . DATE) extracted from PARTS list.
The part whose text contains \"view\" is the view count; the
remaining part is the published date.  Returns empty strings for
absent fields."
  (let (views date)
    (dolist (part parts)
      (let ((text (yeetube-scraper--lockup-part-text part)))
        (cond ((null text))
              ((string-match-p "view" text) (setq views text))
              ((yeetube-scraper--lockup-browse-endpoint part))
              (t (setq date (or date text))))))
    (cons (or views "") (or date ""))))

(defun yeetube-scraper--lockup-duration (renderer)
  "Return duration string from RENDERER's bottom-overlay badge, or nil."
  (let ((overlays (alist-get 'overlays
                             (alist-get 'thumbnailViewModel
                                        (alist-get 'contentImage renderer)))))
    (cl-some (lambda (ov)
               (when-let* ((bot (alist-get 'thumbnailBottomOverlayViewModel ov))
                           (badge (car (alist-get 'badges bot)))
                           (text (alist-get 'text
                                            (alist-get 'thumbnailBadgeViewModel
                                                       badge))))
                 text))
             overlays)))

(defun yeetube-scraper--lockup-thumbnails (renderer)
  "Return thumbnail sources list from a lockupViewModel RENDERER."
  (alist-get 'sources
             (alist-get 'image
                        (alist-get 'thumbnailViewModel
                                   (alist-get 'contentImage renderer)))))

(defun yeetube-scraper--lockup-browse-endpoint (part)
  "Return browseEndpoint alist from lockup metadata PART, or nil."
  (or (alist-get 'browseEndpoint
                 (alist-get 'navigationEndpoint
                            (alist-get 'text part)))
      (alist-get 'browseEndpoint
                 (alist-get 'innertubeCommand
                            (alist-get 'onTap
                                       (car (alist-get 'commandRuns
                                                      (alist-get 'text part))))))))

(defun yeetube-scraper--lockup-channel-identity (renderer)
  "Return (:channel C :channel-id ID :browse-id B) from lockup RENDERER.
Missing fields are empty strings."
  (let* ((meta (alist-get 'lockupMetadataViewModel
                          (alist-get 'metadata renderer)))
         (rows (alist-get 'metadataRows
                          (alist-get 'contentMetadataViewModel
                                     (alist-get 'metadata meta))))
         (channel "")
         (channel-id "")
         (browse-id ""))
    (catch 'found
      (dolist (row rows)
        (dolist (part (alist-get 'metadataParts row))
          (when-let* ((endpoint (yeetube-scraper--lockup-browse-endpoint part)))
            (setq channel (or (yeetube-scraper--lockup-part-text part) "")
                  channel-id (or (alist-get 'canonicalBaseUrl endpoint) "")
                  browse-id (or (alist-get 'browseId endpoint) ""))
            (throw 'found nil)))))
    (list :channel channel :channel-id channel-id :browse-id browse-id)))

(defun yeetube-scraper--extract-video-lockup (renderer)
  "Extract a video plist from a VIDEO-type lockupViewModel RENDERER alist.
YouTube migrated channel-tab video rows from videoRenderer to this
lockup shape in 2025.  Channel identity is filled when metadata parts
carry a browse endpoint; otherwise fields stay empty for page-context
defaults to fill."
  (let* ((id (alist-get 'contentId renderer))
         (title (alist-get 'content
                           (alist-get 'title
                                      (alist-get 'lockupMetadataViewModel
                                                 (alist-get 'metadata renderer)))))
         (parts (yeetube-scraper--lockup-metadata-parts renderer))
         (vd (yeetube-scraper--lockup-views-and-date parts))
         (identity (yeetube-scraper--lockup-channel-identity renderer))
         (thumb-url (yeetube-scraper--thumbnail-url
                     id (yeetube-scraper--lockup-thumbnails renderer))))
    (list :id id
          :title (or title "")
          :views (car vd)
          :duration (or (yeetube-scraper--lockup-duration renderer) "")
          :date (cdr vd)
          :channel (plist-get identity :channel)
          :channel-id (plist-get identity :channel-id)
          :browse-id (plist-get identity :browse-id)
          :thumbnail-url (or thumb-url "")
          :type 'video)))

;;; Playlist extraction (lockupViewModel)

(defun yeetube-scraper--row-text (row)
  "Extract the content string from a metadata ROW."
  (alist-get 'content
	     (alist-get 'text
			(car (alist-get 'metadataParts row)))))

(defun yeetube-scraper--playlist-channel (metadata-rows)
  "Extract channel name from playlist METADATA-ROWS."
  (yeetube-scraper--row-text (car metadata-rows)))

(defun yeetube-scraper--playlist-video-count (metadata-rows)
  "Extract video count string from playlist METADATA-ROWS."
  (when-let* ((found (cl-find-if
                      (lambda (row)
                        (and-let* ((text (yeetube-scraper--row-text row)))
                          (string-match-p "video" text)))
                      metadata-rows)))
    (yeetube-scraper--row-text found)))

(defun yeetube-scraper--playlist-thumbnail (renderer)
  "Extract thumbnail URL from playlist RENDERER."
  (let* ((image (alist-get 'contentImage renderer))
         (collection (alist-get 'collectionThumbnailViewModel image))
         (primary (alist-get 'primaryThumbnail collection))
         (thumb-vm (alist-get 'thumbnailViewModel primary))
         (sources (alist-get 'sources (alist-get 'image thumb-vm))))
    (alist-get 'url (car sources))))

(defun yeetube-scraper--extract-playlist (renderer)
  "Extract a playlist plist from a lockupViewModel RENDERER alist."
  (let* ((meta (alist-get 'lockupMetadataViewModel
			  (alist-get 'metadata renderer)))
         (title (alist-get 'content (alist-get 'title meta)))
         (rows (alist-get 'metadataRows
			  (alist-get 'contentMetadataViewModel
				     (alist-get 'metadata meta))))
         (channel (yeetube-scraper--playlist-channel rows))
         (video-count (yeetube-scraper--playlist-video-count rows))
         (thumb-url (yeetube-scraper--playlist-thumbnail renderer)))
    (list :id (alist-get 'contentId renderer)
          :title (or title "")
          :views ""
          :duration (or video-count "")
          :date ""
          :channel (or channel "")
          :channel-id ""
          :thumbnail-url (or thumb-url "")
          :type 'playlist)))

;;; Item dispatch

(defun yeetube-scraper--dispatch-item (item)
  "Dispatch ITEM to the appropriate extractor.  Return plist or nil."
  (cond
   ((alist-get 'videoRenderer item)
    (yeetube-scraper--extract-video (alist-get 'videoRenderer item)))
   ;; YouTube migrated playlists, and later channel-tab video rows, to
   ;; lockupViewModel.  Branch on contentType.
   ((alist-get 'lockupViewModel item)
    (let ((lockup (alist-get 'lockupViewModel item)))
      (pcase (alist-get 'contentType lockup)
        ("LOCKUP_CONTENT_TYPE_PLAYLIST"
         (yeetube-scraper--extract-playlist lockup))
        ("LOCKUP_CONTENT_TYPE_VIDEO"
         (yeetube-scraper--extract-video-lockup lockup)))))))

;;; Continuation token extraction

(defun yeetube-scraper--extract-continuation (sections)
  "Extract continuation token from SECTIONS.
Return plist (:token T :url U) or nil."
  (let ((cont-item
         (cl-find-if
          (lambda (s) (alist-get 'continuationItemRenderer s))
          sections)))
    (when cont-item
      (let* ((renderer (alist-get 'continuationItemRenderer cont-item))
             (endpoint (alist-get 'continuationEndpoint renderer))
             (token (alist-get 'token
			       (alist-get 'continuationCommand endpoint)))
             (url (alist-get 'apiUrl
			     (alist-get 'webCommandMetadata
					(alist-get 'commandMetadata endpoint)))))
        (when token
          (list :token token :url (or url "")))))))

;;; Section / grid item extraction

(defun yeetube-scraper--extract-section-items (sections)
  "Extract item plists from search result SECTIONS."
  (let* ((item-section
          (cl-find-if
           (lambda (s) (alist-get 'itemSectionRenderer s))
           sections))
         (items (alist-get 'contents
			   (alist-get 'itemSectionRenderer item-section))))
    (cl-loop for item in items
             for plist = (yeetube-scraper--dispatch-item item)
             when plist collect plist)))

(defun yeetube-scraper--extract-grid-items (grid-contents)
  "Extract item plists from channel page GRID-CONTENTS.
Each grid entry is wrapped in a `richItemRenderer'; the inner
`content' is then either a videoRenderer (legacy) or a
lockupViewModel (current YouTube layout), so dispatch through
`yeetube-scraper--dispatch-item' to handle both."
  (cl-loop for entry in grid-contents
           for inner = (alist-get 'content (alist-get 'richItemRenderer entry))
           for plist = (and inner (yeetube-scraper--dispatch-item inner))
           when plist collect plist))

(defun yeetube-scraper--empty-string-p (value)
  "Return non-nil when VALUE is nil or an empty string."
  (or (null value) (and (stringp value) (string-empty-p value))))

(defun yeetube-scraper-fill-channel-identity (items identity)
  "Return ITEMS with empty channel fields filled from IDENTITY plist.
IDENTITY keys are `:channel', `:channel-id', and `:browse-id'.
Nil IDENTITY is a no-op.  Non-empty item fields are preserved."
  (if (null identity)
      items
    (mapcar
     (lambda (item)
       (list :id (plist-get item :id)
             :title (plist-get item :title)
             :views (plist-get item :views)
             :duration (plist-get item :duration)
             :date (plist-get item :date)
             :channel (if (yeetube-scraper--empty-string-p
                           (plist-get item :channel))
                          (or (plist-get identity :channel) "")
                        (plist-get item :channel))
             :channel-id (if (yeetube-scraper--empty-string-p
                              (plist-get item :channel-id))
                             (or (plist-get identity :channel-id) "")
                           (plist-get item :channel-id))
             :browse-id (if (yeetube-scraper--empty-string-p
                             (plist-get item :browse-id))
                            (or (plist-get identity :browse-id) "")
                          (plist-get item :browse-id))
             :thumbnail-url (plist-get item :thumbnail-url)
             :type (plist-get item :type)))
     items)))

(defun yeetube-scraper--vanity-channel-id (url)
  "Return a yeetube channel-id path from vanity URL, or empty string."
  (cond
   ((not (stringp url)) "")
   ((string-match "/\\(@[^/?#]+\\)" url)
    (concat "/" (match-string 1 url)))
   ((string-match "/channel/\\([^/?#]+\\)" url)
    (concat "/channel/" (match-string 1 url)))
   (t "")))

(defun yeetube-scraper--channel-page-identity (json)
  "Return channel identity plist from channel page JSON, or nil."
  (when-let* ((meta (alist-get 'channelMetadataRenderer
                               (alist-get 'metadata json)))
              (browse-id (or (alist-get 'externalId meta) ""))
              (title (or (alist-get 'title meta) ""))
              (channel-id (yeetube-scraper--vanity-channel-id
                           (alist-get 'vanityChannelUrl meta))))
    (when (or (not (string-empty-p browse-id))
              (not (string-empty-p title))
              (not (string-empty-p channel-id)))
      (list :channel title
            :channel-id channel-id
            :browse-id browse-id))))

;;; Page-type parsers

(defun yeetube-scraper--parse-search (contents)
  "Parse search results from CONTENTS alist.
Return plist (:items ITEMS :continuation CONT)."
  (let* ((primary (alist-get 'primaryContents
			     (alist-get 'twoColumnSearchResultsRenderer contents)))
         (sections (alist-get 'contents
			      (alist-get 'sectionListRenderer primary)))
         (items (yeetube-scraper--extract-section-items sections))
         (continuation (yeetube-scraper--extract-continuation sections)))
    (list :items items :continuation continuation)))

(defun yeetube-scraper--parse-channel (contents)
  "Parse channel page from CONTENTS alist.
Return plist (:items ITEMS :continuation CONT)."
  (let* ((tabs (alist-get 'tabs
			  (alist-get 'twoColumnBrowseResultsRenderer contents)))
         (selected (cl-find-if
                    (lambda (tab)
                      (eq t (alist-get 'selected
				       (alist-get 'tabRenderer tab))))
                    tabs))
         (grid-contents
          (thread-last
            (alist-get 'tabRenderer selected)
            (alist-get 'content)
            (alist-get 'richGridRenderer)
            (alist-get 'contents)))
         (items (yeetube-scraper--extract-grid-items grid-contents))
         (continuation (yeetube-scraper--extract-continuation grid-contents)))
    (list :items items :continuation continuation)))

;;; Top-level buffer parser

(defun yeetube-scraper-parse ()
  "Parse ytInitialData from the current buffer.
Return plist (:items ITEM-PLISTS :continuation (:token T :url U)
             [:channel-identity IDENTITY]).
Point is restored after parsing."
  (save-excursion
    (goto-char (point-min))
    (search-forward "ytInitialData")
    (search-forward "=")
    (skip-chars-forward " \t\n")
    (let* ((json (json-parse-buffer :object-type 'alist :array-type 'list))
           (contents (alist-get 'contents json)))
      (cond
       ((alist-get 'twoColumnSearchResultsRenderer contents)
        (yeetube-scraper--parse-search contents))
       ((alist-get 'twoColumnBrowseResultsRenderer contents)
        (let* ((parsed (yeetube-scraper--parse-channel contents))
               (identity (yeetube-scraper--channel-page-identity json)))
          (list :items (yeetube-scraper-fill-channel-identity
                        (plist-get parsed :items) identity)
                :continuation (plist-get parsed :continuation)
                :channel-identity identity)))
       (t (list :items nil :continuation nil))))))

;;; Continuation response parser

(defun yeetube-scraper--continuation-commands (json)
  "Return continuation commands from all supported JSON response roots."
  (cl-loop for key in '(onResponseReceivedCommands
                        onResponseReceivedActions
                        onResponseReceivedEndpoints)
           append (alist-get key json)))

(defun yeetube-scraper--continuation-action (commands)
  "Return first known continuation action alist from COMMANDS."
  (cl-some (lambda (cmd)
             (or (alist-get 'appendContinuationItemsAction cmd)
                 (alist-get 'reloadContinuationItemsCommand cmd)))
           commands))

(defun yeetube-scraper--continuation-container (json)
  "Return a grid or section-list continuation container from JSON."
  (let ((contents (alist-get 'continuationContents json)))
    (or (alist-get 'gridContinuation contents)
        (alist-get 'sectionListContinuation contents))))

(defun yeetube-scraper--container-continuation (container)
  "Extract a continuation plist from CONTAINER, or nil."
  (cl-some
   (lambda (entry)
     (when-let* ((data (or (alist-get 'nextContinuationData entry)
                           (alist-get 'reloadContinuationData entry)))
                 (token (alist-get 'continuation data)))
       (list :token token :url "")))
   (alist-get 'continuations container)))

(defun yeetube-scraper--extract-continuation-items (cont-items)
  "Extract item plists from continuation CONT-ITEMS.
Handles itemSectionRenderer (search), richItemRenderer (channel grid),
and bare videoRenderer / lockupViewModel entries."
  (cl-loop for entry in cont-items
           append
           (cond
            ((alist-get 'itemSectionRenderer entry)
             (yeetube-scraper--extract-section-items (list entry)))
            ((alist-get 'richItemRenderer entry)
             (let* ((inner (alist-get 'content
                                      (alist-get 'richItemRenderer entry)))
                    (plist (and inner (yeetube-scraper--dispatch-item inner))))
               (and plist (list plist))))
            (t
             (let ((plist (yeetube-scraper--dispatch-item entry)))
               (and plist (list plist)))))))

(defun yeetube-scraper-parse-continuation-response (json)
  "Parse a continuation/pagination JSON response.
Return (:items ... :continuation ...)."
  (let* ((commands (yeetube-scraper--continuation-commands json))
         (action (yeetube-scraper--continuation-action commands))
         (container (yeetube-scraper--continuation-container json))
         (cont-items (or (alist-get 'continuationItems action)
                         (alist-get 'items container)
                         (alist-get 'contents container))))
    (list :items (yeetube-scraper--extract-continuation-items cont-items)
          :continuation (or (yeetube-scraper--extract-continuation cont-items)
                            (yeetube-scraper--container-continuation container)))))

(provide 'yeetube-scraper)
;;; yeetube-scraper.el ends here
