Expand on base CLI and fiction handling

.gitignore

@@ -12,6 +12,7 @@
 /config.yml
 
 # Test fiction
+/stories/
 /*.epub
 /*.html
 /*.md

bin/fic_tracker

@@ -2,91 +2,6 @@
 # frozen_string_literal: true
 
 require 'fic_tracker'
-require 'optparse'
+require 'fic_tracker/cli'
-require 'ostruct'
 
-options = OpenStruct.new
+FicTracker::Cli.start
-optparse = OptParse.new do |opts|
-  opts.banner = 'Usage: fic_tracker [OPTIONS...]'
-
-  opts.on '-b', '--backend=BACKEND', 'The backend to use' do |backend|
-    options.backend = backend
-  end
-
-  opts.on '-s', '--story=STORY', 'The story to download' do |story|
-    options.story = story
-  end
-
-  opts.on '-c', '--chapter=CHAPTER', 'The chapter to download' do |chapter|
-    options.chapter = chapter
-  end
-
-  opts.on '-f', '--format=FORMAT', 'The format to download (epub, html, markdown)' do |format|
-    options.format = format.to_sym
-  end
-
-  opts.on '-o', '--output=FILE', 'The resulting file to save the download into' do |output|
-    options.output = output
-  end
-
-  opts.on '--only-changed', 'Only store the story if it has been changed since the last update' do
-    options.only_changed = true
-  end
-
-  opts.separator ''
-
-
-  opts.on '-C', '--config=FILE', 'Specify a configuration file to read' do |config|
-    options.config = config
-  end
-
-  opts.on '-h', '--help', 'Show this text' do
-    puts opts
-    exit
-  end
-
-  opts.on '-q', '--quiet', 'Run quietly' do
-    options.log_level = :error
-  end
-
-  opts.on '-v', '--verbose', 'Run verbosely, can be specified twice for debug output' do
-    if %i[info debug].include?(options.log_level)
-      options.log_level = :debug
-    else
-      options.log_level = :info
-    end
-  end
-
-  opts.on '-V', '--version', 'Prints the version and exits' do
-    puts FicTracker::VERSION
-    exit
-  end
-end
-optparse.parse!
-
-unless options.backend && options.format && options.story
-  puts "Backend, format, and sotry must be provided\n"
-  puts optparse
-  exit 1
-end
-
-FicTracker.logger.level = options.log_level || :warn
-FicTracker.configure!
-
-backend = FicTracker::Backends.get(options.backend)
-
-slug = options.story
-slug = backend.parse_slug(slug) if backend.respond_to? :parse_slug
-
-story = FicTracker::Models::Story.find(backend_name: backend.name, slug:) || FicTracker::Models::Story.new(backend:, slug:)
-before = story.etag
-story.ensure_fully_loaded
-data = nil
-
-options.output ||= "#{story.safe_name}.#{options.format}"
-if !options.only_changed || story.etag != before
-  FicTracker.logger.info "Saving to #{options.output}"
-  File.open(options.output, 'w') { |f| FicTracker::Renderers.render(options.format, story, io: f) }
-end
-
-story.save_changes

config.yml.example

@@ -4,3 +4,9 @@ database:
 
 cache:
   type: database
+
+cli:
+  format: epub
+  output: stories/
+  # Should sync re-render by default
+  render: false

fic_tracker.gemspec

@@ -33,4 +33,5 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'sinatra'
   spec.add_dependency 'sinatra-contrib'
   spec.add_dependency 'sqlite3'
+  spec.add_dependency 'thor'
 end

lib/fic_tracker/backend.rb

@@ -58,6 +58,24 @@ module FicTracker
   end
 
   module Backends
+    def self.find_backend(story_url)
+      # Ensure all backends are loaded
+      Dir[File.dirname(__FILE__) + '/backends/*/backend.rb'].each do |file|
+        require file
+      rescue StandardError => e
+        FicTracker.logger.info "Failed to load backend from #{file}, #{e.class}: #{e}"
+      rescue LoadError => e
+        FicTracker.logger.warn "Failed to load backend from #{file}, #{e.class}: #{e}"
+      end
+
+      constants.each do |b_name|
+        klass = const_get(b_name).const_get :Backend
+        next unless klass.respond_to? :supports_url?
+
+        return b_name if klass.supports_url? story_url
+      end
+    end
+
     def self.get(name)
       const = name
       const = const.to_s.to_sym unless const.is_a? Symbol

lib/fic_tracker/backends/ao3/backend.rb

@@ -23,6 +23,10 @@ module FicTracker::Backends::Ao3
       Client::BASE_URL
     end
 
+    def self.supports_url?(story)
+      story.to_s.start_with? Client::BASE_URL
+    end
+
     def load_author(author)
       author = FicTracker::Models::Author.new(slug: parse_slug(author), backend: self) unless author.is_a? FicTracker::Models::Author
 
@@ -43,7 +47,7 @@ module FicTracker::Backends::Ao3
       image = URI.join(Client::BASE_URL, image[:src]) if image
 
       author.set(
-        name: name,
+        name:,
         url: url&.to_s,
         image: image&.to_s,
         last_metadata_refresh: Time.now
@@ -68,7 +72,7 @@ module FicTracker::Backends::Ao3
       doc = client.request("/works/#{story.slug}/navigate")
 
       chapters = doc.at_css('ol.chapter').css('li').map do |entry|
-        published_at = Time.parse(entry.at_css('span.datetime').text.strip)
+        published_at = Time.from_date_and_time(Time.parse(entry.at_css('span.datetime').text.strip), Time.now)
         link = entry.at_css('a')
         index, *name = link.text.split('. ')
         index = index.to_i
@@ -77,11 +81,11 @@ module FicTracker::Backends::Ao3
         slug = url.path.split('/').last
 
         {
-          slug: slug,
+          slug:,
-          index: index,
+          index:,
-          name: name,
+          name:,
           url: url.to_s,
-          published_at: published_at,
+          published_at:,
         }
       end
 
@@ -161,7 +165,7 @@ module FicTracker::Backends::Ao3
         end
 
         {
-          slug: slug,
+          slug:,
           name: aut_name,
           url: aut[:href],
         }.compact
@@ -175,7 +179,7 @@ module FicTracker::Backends::Ao3
 
           {
             name: a.text.strip,
-            category: category,
+            category:,
             important: %i[rating warning category].include?(category) ? true : nil,
             ordering: TAG_ORDERING[category],
           }.compact
@@ -185,21 +189,21 @@ module FicTracker::Backends::Ao3
       chapters = meta.at_css('dl.stats dd.chapters').text.strip.split('/')
       words = meta.at_css('dl.stats dd.words').text.strip.tr(',', '').to_i
       published_at = meta.at_css('dl.stats dd.published')&.text&.strip
-      published_at = Time.parse(published_at) if published_at
+      published_at = Time.from_date_and_time(Time.parse(published_at), Time.now) if published_at
       updated_at = meta.at_css('dl.stats dd.status')&.text&.strip
-      updated_at = Time.parse(updated_at) if updated_at
+      updated_at = Time.from_date_and_time(Time.parse(updated_at), Time.now) if updated_at
 
       {
-        name: name,
+        name:,
-        authors: authors,
+        authors:,
-        synopsis: synopsis,
+        synopsis:,
         url: url.to_s,
-        language: language,
+        language:,
         chapter_count: chapters.first.to_i,
         word_count: words,
         completed: chapters.first == chapters.last,
-        published_at: published_at,
+        published_at:,
-        updated_at: updated_at,
+        updated_at:,
         tags: FicTracker::Models::Light::Tag.load(tags),
       }
     end
@@ -221,8 +225,8 @@ module FicTracker::Backends::Ao3
       end
 
       {
-        index: index,
+        index:,
-        slug: slug,
+        slug:,
 
         name: title,
         url: url.to_s,

lib/fic_tracker/backends/ao3/client.rb

@@ -52,7 +52,7 @@ module FicTracker::Backends::Ao3
             wait_time = Time.now + wait_time if wait_time.is_a? Numeric
 
             logger.info "Rate limited, waiting until #{wait_time} before retrying"
-            sleep Time.now - wait_time
+            sleep wait_time - Time.now
           else
             break
           end

lib/fic_tracker/cli.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require 'thor'
+
+class FicTracker::Cli < Thor
+  # class_option :config, aliases: :C, default: 'config.yml', desc: 'Specify the configuratino file to use'
+  class_option :quiet, type: :boolean, aliases: :q
+  class_option :verbose, type: :boolean, aliases: :v
+
+  desc 'sync', 'Update all tracked stories'
+  method_option :render, type: :boolean, aliases: :r, desc: 'Render updated stories'
+  def sync
+    setup!
+    should_render = options[:render].nil? ? FicTracker::Config.dig(:cli, :render, default: false) : options[:render]
+    prepare_render! if should_render
+
+    puts "Updating#{ should_render ? ' and rendering' : ''} all stories."
+    FicTracker::Models::Story.each do |story|
+      puts "  Updating #{story} ..."
+      before = story.etag
+      story.ensure_fully_loaded
+      story.save_changes
+      render_story! story if should_render && story.etag != before
+    end
+  end
+
+  desc 'render', 'Render all tracked stories'
+  def render
+    setup!
+    prepare_render!
+
+    puts "Rendering all stories."
+    FicTracker::Models::Story.each do |story|
+      render_story! story
+    end
+  end
+
+  require_relative 'cli/story'
+
+  desc 'story', 'Handle story tracking'
+  subcommand 'story', FicTracker::Cli::Story
+
+  private
+
+  def prepare_render!
+    format = FicTracker::Config.dig(:cli, :format, default: 'epub').to_sym
+    output = FicTracker::Config.dig(:cli, :output, default: 'stories')
+
+    Dir.mkdir output unless Dir.exist? output
+  end
+
+  def render_story!(story)
+    format = FicTracker::Config.dig(:cli, :format, default: 'epub').to_sym
+    output = FicTracker::Config.dig(:cli, :output, default: 'stories')
+
+    filename = File.join output, "#{story.filename}.#{format}"
+    puts "  Rendering #{story} into #{filename} ..."
+
+    File.write(filename, FicTracker::Renderers.render(format, story))
+    File.utime(Time.now, story.updated_at || story.published_at || Time.now, filename)
+  end
+
+  def setup!
+    FicTracker.logger.level = :error if options[:quiet]
+    FicTracker.logger.level = :debug if options[:verbose]
+
+    FicTracker.configure!
+
+    # Ensure overriden log level isn't reset
+    FicTracker.logger.level = :error if options[:quiet]
+    FicTracker.logger.level = :debug if options[:verbose]
+
+    FicTracker.cache.expire
+  end
+end

lib/fic_tracker/cli/story.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+class FicTracker::Cli::Story < Thor
+  desc 'add STORY...', 'Add new stories to tracker'
+  method_option :render, type: :boolean, aliases: :r, desc: 'Render updated stories'
+  def add(*stories)
+    setup!
+    should_render = options[:render].nil? ? FicTracker::Config.dig(:cli, :render, default: false) : options[:render]
+
+    stories.each do |story|
+      puts "Adding #{story} ..."
+      bend = FicTracker::Backends.find_backend story
+      if bend.nil?
+        puts "  Can't track, no available backends."
+        next
+      end
+
+      bend = FicTracker::Backends.get(bend)
+      slug = story
+      slug = bend.parse_slug(story) if bend.respond_to? :parse_slug
+
+      story = FicTracker::Models::Story.find(backend_name: bend.name, slug:)
+      if story
+        puts "  Already tracking."
+        next
+      end
+
+      story = FicTracker::Models::Story.new(backend: bend, slug:)
+      if should_render
+        story.ensure_fully_loaded
+        story.save_changes
+        render_story! story
+      end
+      story.save_changes
+    end
+  end
+
+  desc 'del STORY...', 'Remove stories from tracker'
+  def del(*stories)
+    setup!
+    stories.each do |story|
+      if story.include? '/'
+        backend, slug = story.split '/'
+        backend = nil if backend == '*'
+      else
+        slug = story
+      end
+
+      search = {
+        backend:,
+        slug:
+      }.compact
+      found = FicTracker::Models::Story.where(**search)
+
+      if found.size > 1 && !story.start_with?('*/')
+        puts "Found multiple potential stories for #{story}, please specify which ones to remove using the syntax <backend>/<story>: (use */<story> to remove all)"
+        found.each do |f|
+          puts "  - #{f.backend_name}/#{f.slug} - #{f}"
+        end
+      else
+        found.each do |s|
+          puts "Deleting #{s}"
+          s.destroy
+        end
+      end
+    end
+  end
+
+  desc 'list', 'List all tracked stories'
+  def list
+    setup!
+    info = [['Story', 'Chapters', 'Completed', 'Last Updated'], ['-----','--------','---------','------------']]
+    FicTracker::Models::Story.order_by(:name).each do |story|
+      info << [story.name, story.chapters.size, story.completed?, story.updated_at || story.published_at]
+    end
+    print_table info
+  end
+
+  def self.banner(command, namespace = nil, subcommand = false)
+    "#{basename} #{subcommand_prefix} #{command.usage}"
+  end
+
+  def self.subcommand_prefix
+    self.name.gsub(%r{.*::}, '').gsub(%r{^[A-Z]}) { |match| match[0].downcase }.gsub(%r{[A-Z]}) { |match| "-#{match[0].downcase}" }
+  end
+
+  private
+
+  def setup!
+    FicTracker.logger.level = :error if options[:quiet]
+    FicTracker.logger.level = :debug if options[:verbose]
+
+    FicTracker.configure!
+
+    # Ensure overriden log level isn't reset
+    FicTracker.logger.level = :error if options[:quiet]
+    FicTracker.logger.level = :debug if options[:verbose]
+  end
+
+  def render_story!(story)
+    format = FicTracker::Config.dig(:cli, :format, default: 'epub').to_sym
+    output = FicTracker::Config.dig(:cli, :output, default: 'stories')
+
+    filename = File.join output, "#{story.filename}.#{format}"
+    puts "  Rendering #{story} into #{filename} ..."
+
+    File.write(filename, FicTracker::Renderers.render(format, story))
+    File.utime(Time.now, story.updated_at || story.published_at || Time.now, filename)
+  end
+end

lib/fic_tracker/config.rb

@@ -5,6 +5,7 @@ require 'psych'
 require_relative 'util/cache'
 require_relative 'util/database'
 require_relative 'util/hash_extensions'
+require_relative 'util/time_extensions'
 
 module FicTracker::Config
   class << self
@@ -51,7 +52,7 @@ module FicTracker::Config
     def load_internal
       @config_file = ENV['FT_CONFIG_FILE'] if ENV['FT_CONFIG_FILE']
       begin
-        puts "Loading config #{@config_file}"
+        # puts "Loading config #{@config_file}"
         @config = Psych.load(File.read(@config_file)).deep_transform_keys(&:to_sym)
       rescue StandardError => e
         puts "Failed to load config #{@config_file}, #{e.class}: #{e}"

lib/fic_tracker/migrations/001_create_base.rb

@@ -10,6 +10,7 @@ Sequel.migration do
     create_table(:cache) do
       String :key, null: false, primary_key: true
       File :value, null: true
+      Boolean :expired, null: false, default: false
       DateTime :expire_at, null: true, default: nil
     end
 
@@ -96,6 +97,7 @@ Sequel.migration do
       index %i[slug backend_name], unique: true
 
       String :name, null: false
+      String :filename, null: true, default: nil
       String :synopsis, null: false, text: true
       String :language, null: true, default: 'en'
 

lib/fic_tracker/models/chapter.rb

@@ -23,6 +23,22 @@ module FicTracker::Models
       end
     end
 
+    def allow_upsert!
+      insert_conflict(
+        target: [:story_id, :index],
+        update: {
+          slug: Sequel[:excluded][:slug],
+          etag: Sequel[:excluded][:etag],
+          name: Sequel[:excluded][:name],
+          url: Sequel[:excluded][:url],
+          published_at: Sequel[:excluded][:published_at],
+          updated_at: Sequel[:excluded][:updated_at],
+          last_refresh: Sequel[:excluded][:last_refresh],
+          data: Sequel[:excluded][:data],
+        }
+      )
+    end
+
     def to_s
       return "Chapter #{index}" unless name
 

lib/fic_tracker/models/story.rb

@@ -129,6 +129,7 @@ module FicTracker::Models
         @chapters = nil
         to_add.each do |entry|
           logger.debug "Adding new chapter #{entry.inspect} to story #{self}"
+          entry.allow_upsert!
           add_chapter entry
         end
         if to_remove.any?
@@ -142,6 +143,14 @@ module FicTracker::Models
       end
     end
 
+    def filename
+      super || begin
+        return nil unless backend_name && slug && name
+
+        self.filename = safe_name
+      end
+    end
+
     def ensure_fully_loaded
       return backend.load_full_story(self) unless id
 
@@ -155,6 +164,8 @@ module FicTracker::Models
 
         logger.debug "#{self} - Loaded chapters: #{chapter_loaded}/#{chapter_count}"
 
+        return if chapter_loaded == chapter_count
+
         # Full load if more than two chapters are not loaded
         full_load = (chapter_count - chapter_loaded) > 2
       end
@@ -192,7 +203,7 @@ module FicTracker::Models
     end
 
     def safe_name
-      [backend_name, slug, name].join('_').downcase.gsub(/[^a-z0-9\-_]/, '_').gsub(/__+/, '_')
+      [backend_name, slug, name].join('_').downcase.gsub(/[^a-z0-9\-_]/, '_').gsub(/__+/, '_').gsub(/(^_|_$)/, '')
     end
 
     def refresh_metadata

lib/fic_tracker/renderers.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'nokogiri'
+
 module FicTracker::Renderers
   autoload :Epub, 'fic_tracker/renderers/epub'
   autoload :HTML, 'fic_tracker/renderers/html'
@@ -13,6 +15,8 @@ module FicTracker::Renderers
               HTML
             when :Epub, :epub
               Epub
+            else
+              raise ArgumentError, "Unknown format #{type.inspect}"
             end
 
     stringio = nil

lib/fic_tracker/server/helpers.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module FicTracker
+  class Server
+    module Helpers
+      def backend
+        return env[:ft_backend] if env[:ft_backend]
+        return halt 400, 'No backend provided' unless params['backend']
+
+        env[:ft_backend] = Backends.get params['backend']
+        return halt 400, "Unable to find backend #{params['backend']}" unless env[:ft_backend]
+
+        env[:ft_backend]
+      end
+    end
+  end
+end

lib/fic_tracker/server/story.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+module FicTracker
+  class Server
+    class Story < Sinatra::Base
+      include FicTracker::Server::Helpers
+
+      head '/story/:backend/*.*' do |_backend_name, slug, format|
+        mime = nil
+        case format
+        when 'epub', :epub
+          format = :epub
+          mime = 'application/epub+zip'
+        when 'html', :html
+          format = :html
+          mime = 'text/html'
+        when 'txt', :txt, 'md', :md
+          format = :markdown
+          mime = 'text/markdown'
+        else
+          halt 400, "Unknown format #{format}"
+        end
+
+        content_type mime
+        attachment "#{story.safe_name}.#{format}"
+
+        story = Models::Story.find(backend_name: backend.name, slug:)
+        if story
+          story.set(last_accessed: Time.now)
+
+          last_modified story.updated_at || story.published_at
+          etag story.etag
+        end
+      ensure
+        story&.save_changes
+      end
+
+      # rubocop:disable Metrics/BlockLength
+      get '/story/:backend/*.*' do |_backend_name, slug, format|
+        mime = nil
+        case format
+        when 'epub', :epub
+          format = :epub
+          mime = 'application/epub+zip'
+        when 'html', :html
+          format = :html
+          mime = 'text/html'
+        when 'txt', :txt, 'md', :md
+          format = :markdown
+          mime = 'text/markdown'
+        else
+          halt 400, "Unknown format #{format}"
+        end
+
+        story = Models::Story.find(backend_name: backend.name, slug:)
+        story ||= Models::Story.new(backend_name: backend.name, slug:)
+
+        story.ensure_fully_loaded unless story.chapters&.any?
+        story.set(last_accessed: Time.now)
+
+        content_type mime
+        attachment "#{story.safe_name}.#{format}"
+
+        last_modified story.updated_at || story.published_at
+        etag story.etag
+
+        story.ensure_fully_loaded
+        FicTracker::Renderers.render(format, story)
+      ensure
+        story&.save_changes
+      end
+      # rubocop:enable Metrics/BlockLength
+
+      get '/story/:backend/:slug', provides: :html do |_backend_name, slug|
+        story = Models::Story.find(backend_name: backend.name, slug:)
+        story ||= Models::Story.new(backend_name: backend.name, slug:)
+
+        story.ensure_fully_loaded
+        story.set(last_accessed: Time.now)
+
+        last_modified story.updated_at || story.published_at
+        etag story.etag
+
+        haml :story, format: :html5, locals: { story:, backend: }
+      ensure
+        story&.save_changes
+      end
+
+      get '/story/:backend/:slug/:index', provides: :html do |_backend_name, slug, index|
+        story = Models::Story.find(backend_name: backend.name, slug:)
+        story ||= Models::Story.new(backend_name: backend.name, slug:)
+
+        story.ensure_fully_loaded
+        story.set(last_accessed: Time.now)
+
+        chapter = story.chapters[index.to_i]
+
+        last_modified chapter.published_at
+        etag chapter.etag
+
+        haml :chapter, format: :html5, locals: { chapter:, story:, backend: }
+      ensure
+        story&.save_changes
+      end
+    end
+  end
+end

lib/fic_tracker/util/cache/database.rb

@@ -7,11 +7,12 @@ module FicTracker::Util::CacheImpl
 
     def initialize(table: 'cache', **redis)
       @dataset = FicTracker.database[table.to_s.to_sym]
-      @dataset_live = @dataset.where{ Sequel.|({ expire_at: nil }, Sequel::CURRENT_DATE < expire_at) }
+      @dataset_live = @dataset.where(expired: false)
-      @dataset_expired = @dataset_live.invert
+      @dataset_expired = @dataset.where(expired: true)
     end
 
     def expire
+      @dataset.where { Sequel::CURRENT_DATE >= expire_at }.update(expired: true)
       dataset_expired.delete
     end
 

lib/fic_tracker/util/database.rb

@@ -24,6 +24,8 @@ module FicTracker::Util
       self.migrate(db) if migrate
       db.loggers << Logging.logger[self] if connection_string == :memory
 
+      Sequel::Model.plugin :insert_conflict
+
       db
     end
 

lib/fic_tracker/util/time_extensions.rb

@@ -4,4 +4,8 @@ class Time
   def to_header
     getgm.strftime("%a, %d %b %Y %T GMT")
   end
+
+  def self.from_date_and_time(date, time)
+    Time.new(date.year, date.month, date.day, time.hour, time.min, time.sec, time.utc_offset)
+  end
 end