Add stub for SpaceBattles

lib/fic_tracker/backends/ao3/backend.rb

@@ -50,7 +50,7 @@ module FicTracker::Backends::Ao3
         name:,
         url: url&.to_s,
         image: image&.to_s,
-        last_metadata_refresh: Time.now
+        last_metadata_refresh: Time.now,
       )
     end
 

lib/fic_tracker/backends/ao3/client.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require 'cgi'
 require 'net/http'
 require 'nokogiri'
 require 'json'
@@ -13,7 +12,7 @@ module FicTracker::Backends::Ao3
       URI(BASE_URL)
     end
 
-    def request(path, type: :html, body: nil, query: nil, method: :get, redirect: true)
+    def request(path, type: :html, body: nil, query: nil, method: :get)
       uri = URI.join(url, path)
       uri.query = URI.encode_www_form(query) if query
 

lib/fic_tracker/backends/spacebattles/backend.rb

@@ -0,0 +1,152 @@
+# frozen_string_literal: true
+
+require_relative 'client'
+
+module FicTracker::Backends::Spacebattles
+  class Backend < FicTracker::Backend
+    def client
+      @client ||= Client.new
+    end
+
+    def full_name
+      'SpaceBattles'
+    end
+
+    def url
+      Client::BASE_URL
+    end
+
+    def self.supports_url?(story)
+      story.to_s.start_with? Client::BASE_URL
+    end
+
+    def load_author(author)
+      author = FicTracker::Models::Author.new(slug: parse_slug(author), backend: self) unless author.is_a? FicTracker::Models::Author
+
+      url = "/members/#{author.slug}"
+      page = client.request(url)
+
+      author.set(
+        name: page.at_css('.memberHeader-name').text,
+        url:,
+        image: page.at_css('.memberHeader-avatar img')[:src],
+        last_metadata_refresh: Time.now,
+      )
+    end
+
+    def load_story(story)
+      story = FicTracker::Models::Story.new(slug: parse_slug(story), backend: self) unless story.is_a? FicTracker::Models::Story
+
+      logger.info "Loading story #{story.slug}"
+      page = client.request("/threads/#{story.slug}/reader")
+      attrs = extract_story(page)
+      story.set(
+        last_metadata_refresh: Time.now,
+        **attrs
+      )
+    end
+
+    def find_chapters(story)
+      story = FicTracker::Models::Story.new(slug: parse_slug(story), backend: self) unless story.is_a? FicTracker::Models::Story
+
+      chapters = []
+      client.paginated :query do
+        marks = client.request("/threads/#{story.slug}/threadmarks")
+
+        marks.at_css('div.block-body--threadmarkBody').css('div.structItem').each_with_index do |item, index|
+          main = item.at_css('.structItem-cell--main a')
+          latest = item.at_css('.structItem-cell--main time')
+
+          slug = main[:href].split('-').last
+
+          chapters << {
+            slug:,
+            index:,
+            name: main.text,
+            url: URI.join(url, "/posts/#{slug}/"),
+            published_at: Time.at(latest['data-time'].to_i),
+          }
+        end
+      end
+
+      story.chapters = chapters
+      story.set(
+        last_content_refresh: Time.now,
+      )
+    end
+
+    def load_full_story(story)
+      story = FicTracker::Models::Story.new(slug: parse_slug(story), backend: self) unless story.is_a? FicTracker::Models::Story
+
+      attrs = nil
+      chapters = []
+      client.paginated :path do
+        page = client.request("/threads/#{story.slug}/reader")
+
+        attrs = extract_story(page) if page.at_css('.threadmarkListingHeader-content')
+
+        page.css('article.hasThreadmark').each do |post|
+          chapters << extract_chapter(post)
+        end
+      end
+
+      story.chapters = chapters
+      story.set(
+        last_metadata_refresh: Time.now,
+        last_content_refresh: Time.now,
+        **attrs
+      )
+    end
+
+    def load_chapter(chapter, story)
+      story = load_story(story) unless story.is_a? FicTracker::Models::Story
+      chapter = FicTracker::Models::Chapter.new(slug: parse_slug(chapter), story: story) unless chapter.is_a? FicTracker::Models::Chapter
+
+      page = client.request "/threads/#{story.slug}/#{chapter.slug}"
+      post = page.at_xpath("//article[data-content=\"#{chapter.slug.split('#').last}\"]")
+
+      chapter.set(**extract_chapter(post))
+    end
+
+    def parse_slug(slug)
+      return slug.to_s unless slug.start_with?(url)
+
+      type, *components = slug.sub(url, '').split('/')
+      case type
+      when 'members', 'threads'
+        return components.first
+      end
+
+      slug.to_s
+    end
+
+    private
+
+    def extract_story(page)
+
+      {
+      }
+    end
+
+    def extract_chapter(post)
+      threadmark = post.at_css('span.threadmarkLabel')
+      link = post.at_css('header ul.message-attribution-main a')
+      latest = link.at_css('time')
+
+      html = post.at_css('.message-content article .bbWrapper').to_xml
+
+      {
+        slug: post[:itemid].split('/').last,
+
+        name: threadmark.text,
+        url: post[:itemid],
+        published_at: Time.at(latest['data-time'].to_i),
+        last_refresh: Time.now,
+
+        content: html,
+        content_type: 'text/html',
+        etag: Digest::SHA1.hexdigest(html),
+      }
+    end
+  end
+end

lib/fic_tracker/backends/spacebattles/client.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+require 'net/http'
+require 'nokogiri'
+
+module FicTracker::Backends::Spacebattles
+  class Client
+    BASE_URL = 'https://forums.spacebattles.com'
+
+    def initialize
+      @pagination_query = nil
+      @pagination_path = nil
+    end
+
+    def url
+      URI(BASE_URL)
+    end
+
+    def request(path, query: nil, method: :get)
+      path = File.join(path, @pagination_path) if @pagination_path
+      uri = URI.join(url, path)
+      query = @pagination_query.merge query if @pagination_query
+      uri.query = URI.encode_www_form(query) if query
+
+      req = Net::HTTP.const_get(method.to_s.capitalize.to_sym).new uri.request_uri
+      req['User-Agent'] = "FicTracker/#{FicTracker::VERSION}"
+      req['Accept'] = 'text/html'
+
+      resp = nil
+      http.start do
+        loop do
+          debug_http(req)
+          resp = http.request req
+          debug_http(resp)
+          case resp
+          when Net::HTTPRedirection
+            uri = URI.join(url, resp['location'])
+            uri.query = URI.encode_www_form(query) if query
+            req.path.replace uri.request_uri
+          when Net::HTTPTooManyRequests
+            wait_time = 10
+            if resp['retry-after']
+              after = resp['retry-after']
+              wait_time = after =~ /\A[0-9]+\Z/ ? after.to_i : Time.parse(after)
+            end
+            wait_time = Time.now + wait_time if wait_time.is_a? Numeric
+
+            logger.info "Rate limited, waiting until #{wait_time} before retrying"
+            sleep wait_time - Time.now
+          else
+            break
+          end
+        end
+      end
+
+      resp.value
+      return if resp.body.empty?
+
+      Nokogiri::HTML4.parse(resp.body)
+    end
+
+    def paginated(style, per_page: 200, &_block)
+      raise 'Style must be :path or :query' unless %i[path query].include? style
+      @pagination_query = { per_page: per_page } if style == :query
+      @pagination_path = '' if style == :path
+      page = 1
+
+      loop do
+        result = yield
+
+        return result unless result.at_css('nav.pageNavWrapper .pageNav-jump--next')
+
+        page += 1
+        @pagination_query[:page] = page if style == :query
+        @pagination_path = "page-#{page}" if style == :path
+      end
+    ensure
+      @pagination_query = nil
+      @pagination_path = nil
+    end
+
+    private
+
+    def logger
+      Logging.logger[self]
+    end
+
+    def http
+      @http ||= Net::HTTP.new(url.host, url.port).tap { |http| http.use_ssl = url.scheme == 'https' }
+    end
+
+    def debug_http(object)
+      return unless logger.debug?
+
+      dir = '>'
+      if object.is_a?(Net::HTTPRequest)
+        dir = '<'
+
+        logger.debug "#{dir} #{object.method} #{object.path}"
+      else
+        logger.debug "#{dir} #{object.code} #{object.message}"
+      end
+
+      object.each_header { |h, v| logger.debug "#{dir} #{h}: #{v}" }
+    end
+  end
+end
+

lib/fic_tracker/cli.rb

@@ -9,13 +9,18 @@ class FicTracker::Cli < Thor
 
   desc 'sync', 'Update all tracked stories'
   method_option :render, type: :boolean, aliases: :r, desc: 'Render updated stories'
+  method_option :cron, type: :boolean, desc: 'Limit simultaenous actions, to reduce load when running as a cron job'
   def sync
     setup!
     should_render = options[:render].nil? ? FicTracker::Config.dig(:cli, :render, default: false) : options[:render]
     prepare_render! if should_render
 
     puts "Updating#{ should_render ? ' and rendering' : ''} stories."
-    FicTracker::Models::Story.needing_content_refresh.each do |story|
+    to_update = FicTracker::Models::Story.needing_content_refresh
+    to_update = to_update.to_a.sample 10 if options[:cron]
+    to_update.each do |story|
+      next unless story.needs_content_refresh? || story.needs_metadata_refresh?
+
       puts "  Updating #{story} ..."
       before = story.etag
       story.ensure_fully_loaded

lib/fic_tracker/cli/story.rb

@@ -35,6 +35,28 @@ class FicTracker::Cli::Story < Thor
     end
   end
 
+  desc 'download STORY...', 'Download stories without adding them to the tracker'
+  def download(*stories)
+    setup!
+
+    stories.each do |story|
+      puts "Downloading #{story} ..."
+      bend = FicTracker::Backends.find_backend story
+      if bend.nil?
+        puts "  Can't download, no available backends."
+        next
+      end
+
+      bend = FicTracker::Backends.get(bend)
+      slug = story
+      slug = bend.parse_slug(story) if bend.respond_to? :parse_slug
+
+      story = FicTracker::Models::Story.new(backend: bend, slug:)
+      story.ensure_fully_loaded
+      render_story! story
+    end
+  end
+
   desc 'del STORY...', 'Remove stories from tracker'
   def del(*stories)
     setup!
@@ -71,7 +93,7 @@ class FicTracker::Cli::Story < Thor
     setup!
     info = [['Slug', 'Story', 'Chapters', 'Completed', 'Last Updated'], ['----','-----','--------','---------','------------']]
     FicTracker::Models::Story.order_by(:name).each do |story|
-      info << [story.slug, story.name, story.chapters.size, story.completed?, (story.updated_at || story.published_at).strftime("%F")]
+      info << ["#{story.backend_name}/#{story.slug}", story.name, story.chapters.size, story.completed?, (story.updated_at || story.published_at).strftime("%F")]
     end
     print_table info
   end

lib/fic_tracker/migrations/001_create_base.rb

@@ -120,7 +120,7 @@ Sequel.migration do
       String :data, null: false, text: true, default: '{}'
     end
 
-    if false
+    if false # multi_user
       create_table(:users) do
         primary_key :id
 

lib/fic_tracker/models/story.rb

@@ -158,30 +158,23 @@ module FicTracker::Models
       refresh_content(splay: true)
       refresh_metadata(splay: true)
 
-      full_load = true
       if chapters && chapters.size > 0
-        chapter_count = chapters.size
+        partial_load_chapters
-        chapter_loaded = chapters.count { |c| c.content? && c.content_type? }
+        return self
-
-        logger.debug "#{self} - Loaded chapters: #{chapter_loaded}/#{chapter_count}"
-
-        return if chapter_loaded == chapter_count
-
-        # Full load if more than two chapters are not loaded
-        full_load = (chapter_count - chapter_loaded) > 2
       end
 
-      if full_load
+      logger.debug "#{self} - Performing full load"
-        logger.debug "#{self} - Performing full load"
+      backend.load_full_story(self)
-        backend.load_full_story(self)
+
-      else
+      self
-        logger.debug "#{self} - Ensuring all chapters are loaded"
-        chapters.each(&:content)
-      end
     end
 
     def ensure_chapters
-      ensure_fully_loaded
+      return ensure_fully_loaded unless id
+      return ensure_fully_loaded if chapters.nil? || chapters.size.zero?
+      return ensure_fully_loaded if needs_content_refresh? || needs_metadata_refresh?
+
+      partial_load_chapters
     end
 
     def backend
@@ -223,7 +216,7 @@ module FicTracker::Models
       return unless backend && needs_content_refresh?
 
       self.last_content_refresh += rand(-3600..3600) if splay
-      refresh_content! 
+      refresh_content!
       # chapters.each(&:refresh_content)
     end
 
@@ -279,5 +272,24 @@ module FicTracker::Models
     def logger
       Logging.logger[self]
     end
+
+    def partial_load_chapters
+      chapter_count = chapters.size
+      chapter_loaded = chapters.count { |c| c.content? && c.content_type? }
+
+      return false if chapter_loaded == chapter_count
+
+      full_load = (chapter_count - chapter_loaded) > 2
+
+      if full_load
+        logger.debug "#{self} - Performing full load"
+        backend.load_full_story(self)
+      else
+        logger.debug "#{self} - Ensuring all chapters are loaded"
+        chapters.each(&:content)
+      end
+
+      true
+    end
   end
 end

lib/fic_tracker/renderers/epub.rb

@@ -19,6 +19,9 @@ module FicTracker::Renderers
     def render
       logger.info "Rendering epub for #{story}"
 
+      story.ensure_chapters
+      story.save_changes if story.id
+
       require 'zip'
       require 'zip/filesystem'
 

lib/fic_tracker/renderers/html.rb

@@ -17,6 +17,9 @@ module FicTracker::Renderers
     def render
       logger.info "Rendering html for #{story}"
 
+      story.ensure_chapters
+      story.save_Changes if story.id
+
       doc = Nokogiri::HTML5::Document.new
       Nokogiri::XML::Builder.with(doc) do |html|
         build_html(html) do

lib/fic_tracker/renderers/markdown.rb

@@ -16,6 +16,9 @@ module FicTracker::Renderers
     def render
       logger.info "Rendering markdown for #{story}"
 
+      story.ensure_chapters
+      story.save_changes if story.id
+
       @io.puts build_preface, nil if preface
 
       story.chapters.each do |chapter|