Improve db cache and support conditional render

.editorconfig

@@ -0,0 +1,14 @@
+# EditorConfig is awesome: http://EditorConfig.org
+
+# top-most EditorConfig file
+root = true
+
+[*]
+end_of_line = lf
+charset = utf-8
+
+indent_size = 2
+indent_style = space
+
+trim_trailing_whitespace = true
+insert_final_newline = true

.gitignore

@@ -10,3 +10,8 @@
 /Gemfile.lock
 /database.db
 /config.yml
+
+# Test fiction
+/*.epub
+/*.html
+/*.md

Gemfile

@@ -10,3 +10,5 @@ gem "rake", "~> 13.0"
 gem "minitest", "~> 5.16"
 
 gem "rubocop", "~> 1.21"
+
+gem 'irb'

bin/fic_tracker

@@ -6,9 +6,9 @@ require 'optparse'
 require 'ostruct'
 
 options = OpenStruct.new
-OptParse.new do |opts|
+optparse = OptParse.new do |opts|
   opts.banner = 'Usage: fic_tracker [OPTIONS...]'
-  
+
   opts.on '-b', '--backend=BACKEND', 'The backend to use' do |backend|
     options.backend = backend
   end
@@ -29,8 +29,13 @@ OptParse.new do |opts|
     options.output = output
   end
 
+  opts.on '--only-changed', 'Only store the story if it has been changed since the last update' do
+    options.only_changed = true
+  end
+
   opts.separator ''
 
+
   opts.on '-C', '--config=FILE', 'Specify a configuration file to read' do |config|
     options.config = config
   end
@@ -56,18 +61,32 @@ OptParse.new do |opts|
     puts FicTracker::VERSION
     exit
   end
-end.parse!
+end
+optparse.parse!
+
+unless options.backend && options.format && options.story
+  puts "Backend, format, and sotry must be provided\n"
+  puts optparse
+  exit 1
+end
 
 FicTracker.logger.level = options.log_level || :warn
 FicTracker.configure!
 
 backend = FicTracker::Backends.get(options.backend)
-story = FicTracker::Models::Story.find(backend_name: backend.name, slug: backend.parse_slug(options.story)) || FicTracker::Models::Story.new(backend: backend, slug: backend.parse_slug(options.story))
+
+slug = options.story
+slug = backend.parse_slug(slug) if backend.respond_to? :parse_slug
+
+story = FicTracker::Models::Story.find(backend_name: backend.name, slug:) || FicTracker::Models::Story.new(backend:, slug:)
+before = story.etag
 story.ensure_fully_loaded
 data = nil
 
 options.output ||= "#{story.safe_name}.#{options.format}"
-FicTracker.logger.info "Saving to #{options.output}"
+if !options.only_changed || story.etag != before
-File.open(options.output, 'w') { |f| FicTracker::Renderers.render(options.format, story, io: f) }
+  FicTracker.logger.info "Saving to #{options.output}"
+  File.open(options.output, 'w') { |f| FicTracker::Renderers.render(options.format, story, io: f) }
+end
 
 story.save_changes

config.ru

@@ -1,7 +1,15 @@
 # frozen_string_literal: true
 
 require 'fic_tracker'
+require 'fic_tracker/server'
 
 FicTracker.configure!
 
-map '/' ->() { run FicTracker::Server }
+map '/health' do
+  run -> { [200, { 'Content-Type' => 'text/plain' }, ['OK']] }
+end
+
+server = FicTracker::Server.new
+map '/' do
+  run server
+end

fic_tracker.gemspec

@@ -31,5 +31,6 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'rubyzip'
   spec.add_dependency 'sequel'
   spec.add_dependency 'sinatra'
+  spec.add_dependency 'sinatra-contrib'
   spec.add_dependency 'sqlite3'
 end

lib/fic_tracker.rb

@@ -51,50 +51,9 @@ module FicTracker
     @global_logger ||= false
   end
 
-  module Models
+  autoload :Models, 'fic_tracker/models'
-    def self.const_missing(const)
+  autoload :Renderers, 'fic_tracker/renderers'
-      raise 'No database connected' unless FicTracker.database
+  autoload :Tasks, 'fic_tracker/tasks'
-
-      model = const.to_s.downcase
-      require_relative "fic_tracker/models/#{model}"
-
-      mod = const_get(const) if const_defined? const
-      return mod if mod
-
-      raise "Model not found: #{const}"
-    end
-  end
-
-  module Tasks
-    autoload :Cleanup, 'fic_tracker/tasks/cleanup'
-  end
-
-  module Renderers
-    autoload :Epub, 'fic_tracker/renderers/epub'
-    autoload :HTML, 'fic_tracker/renderers/html'
-    autoload :Markdown, 'fic_tracker/renderers/markdown'
-
-    def self.render(type, story, **attrs)
-      klass = case type
-              when :Markdown, :markdown, :md
-                Markdown
-              when :HTML, :html
-                HTML
-              when :Epub, :epub
-                Epub
-              end
- 
-      unless attrs[:io]
-        require 'stringio'
-        attrs[:io] = StringIO.new
-        attrs[:_stringio] = true
-      end
-
-      return klass.new(story, **attrs).render unless attrs[:stringio]
-
-      attrs[:io].string
-    end
-  end
 end
 
 require_relative 'fic_tracker/backend'

lib/fic_tracker/backend.rb

@@ -86,7 +86,7 @@ module FicTracker
       return unless model
 
       backend = const_get(model).const_get(:Backend)
-      backend.new(**FicTracker.config.dig(:backends, backend.config_name, default: {}))
+      backend.new(**FicTracker::Config.dig(:backends, backend.config_name, default: {}))
     rescue StandardError => e
       Logging.logger[backend].error "Failed to load, #{e.class}: #{e}"
 

lib/fic_tracker/backends/ao3/backend.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative 'client'
+require_relative 'search_info'
 
 module FicTracker::Backends::Ao3
   class Backend < FicTracker::Backend
@@ -9,7 +10,7 @@ module FicTracker::Backends::Ao3
       category: -2,
       warning:  -1,
     }.freeze
-  
+
     def client
       @client ||= Client.new
     end
@@ -25,20 +26,26 @@ module FicTracker::Backends::Ao3
     def load_author(author)
       author = FicTracker::Models::Author.new(slug: parse_slug(author), backend: self) unless author.is_a? FicTracker::Models::Author
 
-      logger.info "Loading author #{author.slug}"
+      logger.info "Loading author #{author}"
-      doc = client.request("/users/#{author.slug}")
+      slug, pseud = author.slug.split('/')
+      pseud ||= slug
+
+      doc = client.request("/users/#{slug}/pseuds/#{pseud}")
 
       user = doc.at_css('#main .user')
 
+      name = user.at_css('h2.heading').text.sub(/\([^)]+\)$/, '').strip
+      name = nil if name == pseud
+
       url = user.at_css('.icon a')
       url = URI.join(Client::BASE_URL, url[:href]) if url
       image = user.at_css('.icon a img')
       image = URI.join(Client::BASE_URL, image[:src]) if image
 
       author.set(
-        name: user.at_css('h2.heading').text.strip,
+        name: name,
-        url: url.to_s,
+        url: url&.to_s,
-        image: image.to_s,
+        image: image&.to_s,
         last_metadata_refresh: Time.now
       )
     end
@@ -47,7 +54,7 @@ module FicTracker::Backends::Ao3
       story = FicTracker::Models::Story.new(slug: parse_slug(story), backend: self) unless story.is_a? FicTracker::Models::Story
 
       logger.info "Loading story #{story.slug}"
-      doc = client.request("/works/#{story.slug}")
+      doc = client.request("/works/#{story.slug}", query: { view_adult: true })
 
       attrs = extract_story doc
 
@@ -90,7 +97,7 @@ module FicTracker::Backends::Ao3
       story = FicTracker::Models::Story.new(slug: parse_slug(story), backend: self) unless story.is_a? FicTracker::Models::Story
 
       logger.info "Loading all chapters for #{story.slug}"
-      doc = client.request("/works/#{story.slug}", query: { view_full_work: true })
+      doc = client.request("/works/#{story.slug}", query: { view_full_work: true, view_adult: true })
 
       attrs = extract_story(doc)
       chapters = doc.css('#chapters > div.chapter').map { |chapter| extract_chapter(chapter) }
@@ -110,13 +117,23 @@ module FicTracker::Backends::Ao3
       chapter = FicTracker::Models::Chapter.new(slug: parse_slug(chapter), story: story) unless chapter.is_a? FicTracker::Models::Chapter
 
       logger.info "Loading chapter #{chapter.slug} for #{story.slug}"
-      doc = client.request("/works/#{story.slug}/chapters/#{chapter.slug}")
+      doc = client.request("/works/#{story.slug}/chapters/#{chapter.slug}", query: { view_adult: true })
 
       attrs = extract_chapter(doc.at_css('#chapters > div.chapter'))
 
       chapter.set(**attrs)
     end
 
+    def get_search_info
+      SearchInfo.new(self)
+    end
+
+    def search(search_info)
+      info = SearchInfo.from_search(search_info)
+
+      info
+    end
+
     def parse_slug(slug)
       return URI(slug).path.split('/').last if slug.is_a?(String) && slug.start_with?('http')
 
@@ -131,9 +148,24 @@ module FicTracker::Backends::Ao3
       preface = doc.at_css('#workskin .preface')
 
       name = preface.at_css('.title').text.strip
-      synopsis = preface.at_css('.summary blockquote').children.to_xml
+      synopsis = preface.at_css('.summary blockquote').children.to_xml.strip
       language = meta.at_css('dd.language')['lang']
-      author = preface.at_css('a[rel="author"]')[:href].split('/')[2]
+      authors = preface.css('a[rel="author"]').map do |aut|
+        slug = aut[:href].split('/')[2]
+        pseud = aut[:href].split('/').last
+
+        aut_name = nil
+        if slug != pseud
+          aut_name = pseud
+          slug = "#{slug}/#{pseud}"
+        end
+
+        {
+          slug: slug,
+          name: aut_name,
+          url: aut[:href],
+        }.compact
+      end
 
       tags = meta.css('dd.tags').map do |tagblock|
         category = tagblock[:class].split.first.to_sym
@@ -156,15 +188,15 @@ module FicTracker::Backends::Ao3
       published_at = Time.parse(published_at) if published_at
       updated_at = meta.at_css('dl.stats dd.status')&.text&.strip
       updated_at = Time.parse(updated_at) if updated_at
-      
+
       {
         name: name,
-        author: author,
+        authors: authors,
         synopsis: synopsis,
         url: url.to_s,
         language: language,
-        chapter_count: chapters.first.to_i, 
+        chapter_count: chapters.first.to_i,
-        word_count: words, 
+        word_count: words,
         completed: chapters.first == chapters.last,
         published_at: published_at,
         updated_at: updated_at,
@@ -188,7 +220,7 @@ module FicTracker::Backends::Ao3
         title = title_extra.empty? ? title_base : title_extra
       end
 
-      { 
+      {
         index: index,
         slug: slug,
 
@@ -198,6 +230,7 @@ module FicTracker::Backends::Ao3
 
         content: html,
         content_type: 'text/html',
+        etag: Digest::SHA1.hexdigest(html),
       }
     end
   end

lib/fic_tracker/backends/ao3/client.rb

@@ -40,7 +40,9 @@ module FicTracker::Backends::Ao3
           debug_http(resp)
           case resp
           when Net::HTTPRedirection
-            req.path.replace resp['location']
+            uri = URI.join(url, resp['location'])
+            uri.query = URI.encode_www_form(query) if query
+            req.path.replace uri.request_uri
           when Net::HTTPTooManyRequests
             wait_time = 10
             if resp['retry-after']

lib/fic_tracker/backends/ao3/search_info.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require 'fic_tracker/models/light/search_info'
+
+module FicTracker::Backends::Ao3
+  class SearchInfo < FicTracker::Models::Light::SearchInfo
+    # Meta tags
+    tag_category :category, list: [
+      {name: 'Gen'},
+      {}
+    ]
+    tag_category :rating, list: [
+      {},
+      {}
+    ]
+
+    tag_category :character, freeform: true
+    tag_category :fandom, search: ->(search) { autocomplete_fandom(search) }
+    tag_category :relationship, freeform: true
+    tag_category :freeform, freeform: true
+
+    def initialize(backend)
+      @backend = backend
+      @client = backend.client
+    end
+
+    private
+
+    def autocomplete_fandom(name)
+      @client.request("/autocomplete").map do |tag|
+      end
+    end
+  end
+end

lib/fic_tracker/config.rb

@@ -4,6 +4,7 @@ require 'psych'
 
 require_relative 'util/cache'
 require_relative 'util/database'
+require_relative 'util/hash_extensions'
 
 module FicTracker::Config
   class << self
@@ -24,6 +25,7 @@ module FicTracker::Config
         encoding: dig(:cache, :encoding, default: nil),
         options: dig(:cache, :options, default: {})
       )
+      FicTracker.logger.level = dig(:log, :level, default: :info)
 
       Sequel::Model.db = FicTracker.database
     end
@@ -49,8 +51,10 @@ module FicTracker::Config
     def load_internal
       @config_file = ENV['FT_CONFIG_FILE'] if ENV['FT_CONFIG_FILE']
       begin
+        puts "Loading config #{@config_file}"
         @config = Psych.load(File.read(@config_file)).deep_transform_keys(&:to_sym)
-      rescue
+      rescue StandardError => e
+        puts "Failed to load config #{@config_file}, #{e.class}: #{e}"
         @config = {}
       end
       @config[:database] ||= {}

lib/fic_tracker/migrations/001_create_base.rb

@@ -19,7 +19,7 @@ Sequel.migration do
       String :backend_name, null: false
       index %i[slug backend_name], unique: true
 
-      String :name, null: false
+      String :name, null: true, default: nil
 
       String :url, null: true, default: nil
       String :image, null: true, default: nil
@@ -30,6 +30,14 @@ Sequel.migration do
       String :data, null: false, text: true, default: '{}'
     end
 
+    create_table(:authors_stories) do
+      primary_key :id
+
+      foreign_key :author_id, :authors, on_delete: :cascade, on_update: :cascade
+      foreign_key :story_id, :stories, on_delete: :cascade, on_update: :cascade
+      index %i[author_id story_id], unique: true
+    end
+
     create_table(:chapters) do
       primary_key :id
 
@@ -76,6 +84,8 @@ Sequel.migration do
 
       foreign_key :collection_id, :collections, on_delete: :cascade, on_update: :cascade
       foreign_key :story_id, :stories, on_delete: :cascade, on_update: :cascade
+      index %i[collection_id story_id], unique: true
+
       Integer :index, null: true, default: nil
     end
 
@@ -85,8 +95,6 @@ Sequel.migration do
       String :backend_name, null: false
       index %i[slug backend_name], unique: true
 
-      foreign_key :author_id, :authors, on_delete: :cascade, on_update: :cascade
-
       String :name, null: false
       String :synopsis, null: false, text: true
       String :language, null: true, default: 'en'

lib/fic_tracker/models.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module FicTracker::Models
+  def self.const_missing(const)
+    raise 'No database connected' unless FicTracker.database
+
+    model = const.to_s.downcase
+    require_relative "models/#{model}"
+
+    mod = const_get(const) if const_defined? const
+    return mod if mod
+
+    raise "Model not found: #{const}"
+  end
+end

lib/fic_tracker/models/author.rb

@@ -2,12 +2,12 @@
 
 module FicTracker::Models
   class Author < Sequel::Model
-    # 1/week
+    # 2/month
-    METADATA_REFRESH_INTERVAL = 7 * 24 * 60 * 60
+    METADATA_REFRESH_INTERVAL = 14 * 24 * 60 * 60
 
     plugin :serialization, :json, :data
 
-    one_to_many :stories
+    many_to_many :stories
 
     def before_create
       backend.load_author(self) if last_metadata_refresh.nil?
@@ -15,6 +15,11 @@ module FicTracker::Models
       super
     end
 
+    def self.expire
+      # TODO: Improve
+      all.select { |a| a.stories.empty? }.each(&:destroy)
+    end
+
     def backend
       return unless backend_name
 
@@ -40,9 +45,14 @@ module FicTracker::Models
       Time.now - (last_metadata_refresh || Time.at(0)) >= METADATA_REFRESH_INTERVAL
     end
 
+    def self.needing_metadata_refresh
+      where { Sequel.|(last_metadata_refresh.nil?, last_metadata_refresh < date.function(Time.now - METADATA_REFRESH_INTERVAL, 'localtime')) }
+    end
+
     def to_s
-      name || slug
+      return name if name
+
+      slug
     end
   end
 end
-

lib/fic_tracker/models/chapter.rb

@@ -6,12 +6,23 @@ module FicTracker::Models
   class Chapter < Sequel::Model
     # 3/day
     CONTENT_REFRESH_INTERVAL = 12 * 60 * 60
-    CONTENT_CACHE_TIME = 7 * 24 * 60 * 60
+    CONTENT_CACHE_TIME = 14 * 24 * 60 * 60
 
     plugin :serialization, :json, :data
 
     many_to_one :story
 
+    def after_create
+      if @content
+        key = cache_key + [:content]
+        FicTracker.cache.set(key, @content, CONTENT_CACHE_TIME)
+      end
+      if @content_type
+        key = cache_key + [:content_type]
+        FicTracker.cache.set(key, @content_type, CONTENT_CACHE_TIME)
+      end
+    end
+
     def to_s
       return "Chapter #{index}" unless name
 
@@ -35,9 +46,10 @@ module FicTracker::Models
       return unless cache_key
 
       key = cache_key + [:content]
-      refresh_content! unless FicTracker.cache.has?(key)
-
       @content ||= FicTracker.cache.get(key)
+      refresh_content! unless @content
+
+      @content
     end
 
     def content=(content)
@@ -46,7 +58,6 @@ module FicTracker::Models
         FicTracker.cache.set(key, content, CONTENT_CACHE_TIME)
       end
       @content = content
-      etag = Digest::SHA1.hexdigest(content)
     end
 
     def content_type?
@@ -62,9 +73,10 @@ module FicTracker::Models
       return unless cache_key
 
       key = cache_key + [:content_type]
-      refresh_content! unless FicTracker.cache.has?(key)
-
       @content_type ||= FicTracker.cache.get(key)
+      refresh_content! unless @content_type
+
+      @content_type
     end
 
     def content_type=(type)

lib/fic_tracker/models/light/search_info.rb

@@ -1,8 +1,17 @@
 # frozen_string_literal: true
 
-
 module FicTracker::Models::Light
   class SearchInfo
+    def initialize(**data)
+      data.each do |k, v|
+        send(:"#{k}=", v) if respond_to?(:"#{k}=")
+      end
+    end
+
+    def with_language(lang)
+      self
+    end
+
     def with_tag(tag)
       self
     end
@@ -15,10 +24,56 @@ module FicTracker::Models::Light
       self
     end
 
+    def with_cursor(cursor)
+      self
+    end
+
+    def find_tags(category, search = nil)
+      raise ArgumentError, "No such category #{category.inspect}" unless @tags.key? category
+
+      tag_cat = @tags[category]
+      return if tag_cat[:freeform]
+
+      list = begin
+        if tag_cat[:list]
+          return tag_cat[:list] if search.nil?
+          return tag_cat[:list].select do |t|
+            case search
+            when Regex
+              t[:name] =~ search
+            when String
+              t[:name].downcase.include? search.downcase
+            else
+              false
+            end
+          end
+        end
+        return tag_cat[:search].call(search) if tag_cat[:search]
+      end
+      list.each do |t|
+        t[:id] ||= t[:name]
+        t[:category] ||= category
+      end
+      FicTracker::Models::Light::Tag.load list
+    end
+
+    def to_info_json
+      self.class.to_info_json
+    end
+
     class << self
+      def from_search(data)
+
+      end
+
+      def to_info_json
+
+      end
+
       protected
-      
+
       def tag_category(category, **params)
+        (@tags ||= {})[category] = params
       end
 
       def word_limits(*limits)

lib/fic_tracker/models/light/tag.rb

@@ -11,7 +11,7 @@ module FicTracker::Models::Light
     end
 
     def self.store(data)
-      JSON.generate(data)
+      data = JSON.generate(data)
     end
 
     attr_accessor :id, :name, :category

lib/fic_tracker/models/story.rb

@@ -6,31 +6,22 @@ module FicTracker::Models
   class Story < Sequel::Model
     # 1/week
     METADATA_REFRESH_INTERVAL = 7 * 24 * 60 * 60
-    # 3/day
+    # 6/day
-    CONTENT_REFRESH_INTERVAL = 12 * 60 * 60
+    CONTENT_REFRESH_INTERVAL = 4 * 60 * 60
+    # 2 months
+    STORY_EXPIRY = 2 * 30 * 24 * 60 * 60
 
-    many_to_one :author
+    many_to_many :authors
     one_to_many :chapters, order: :index
     many_to_many :collection, join_table: :collection_stories
 
     plugin :serialization, [Light::Tag.method(:store), Light::Tag.method(:load)], :tags
     plugin :serialization, :json, :data
 
-    # Defer creation of author/chapters until story requiring them is to be saved
+    # Defer creation of authors/chapters until the story requiring them is to be saved
-    def before_create
-      if @author
-        @author.save unless @author.id
-        self.author_id = @author.id
-      end
-
-      super
-    end
-
     def after_create
-      return if [@author, @chapters].all?(&:nil?)
+      @authors&.each { |author| add_author author }
-
+      @authors = nil
-      self.author = @author if @author
-      @author = nil
 
       if @chapters
         latest_chapter_at = self.updated_at || self.published_at || Time.at(0)
@@ -43,13 +34,21 @@ module FicTracker::Models
       @chapters = nil
     end
 
+    def self.expire
+      where { last_accessed < date.function(Time.now - FicTracker::Models::Story::STORY_EXPIRY, 'localtime') }.destroy
+    end
+
     def completed?
       completed
     end
 
     # Support attaching author to a not-yet-saved story
+    def authors
+      @authors || super
+    end
+
     def author
-      @author || super
+      authors.first
     end
 
     def author=(author_name)
@@ -57,13 +56,45 @@ module FicTracker::Models
       author ||= Author.find(backend_name: backend.name, slug: author_name)
       author ||= backend.load_author(author_name) if id
       author ||= Author.new(backend: backend, slug: author_name)
-      
+
+      self.authors = [author]
+    end
+
+    def authors=(authors)
+      to_add = []
+      to_remove = self.authors.map(&:id)
+
+      authors.each do |entry|
+        aut = entry if entry.is_a?(FicTracker::Models::Author)
+
+        if aut
+          to_add << aut
+        else
+          aut = self.authors.find { |c| c.slug == entry[:slug] }
+
+          if aut
+            aut.set(**entry)
+          else
+            entry[:backend_name] = backend.name
+            aut = FicTracker::Models::Author.new(**entry)
+            to_add << aut
+          end
+        end
+        to_remove.delete aut.id
+      end
+
       if id
-        @author = nil
+        @authors = nil
-        author.save unless author.id
+        to_add.each do |entry|
-        super(author)
+          logger.debug "Adding author #{entry.inspect} to story #{self}"
+          add_author entry
+        end
+        if to_remove.any?
+          logger.debug "Removing author(s) #{to_remove.inspect} from story #{self}"
+          author_dataset.where(id: to_remove).destroy
+        end
       else
-        @author = author
+        @authors = (@authors || []) + to_add - to_remove
       end
     end
 
@@ -71,7 +102,7 @@ module FicTracker::Models
     def chapters
       @chapters || super
     end
-    
+
     def chapters=(entries)
       latest_chapter_at = self.published_at || Time.at(0)
 
@@ -80,7 +111,7 @@ module FicTracker::Models
 
       entries.each do |entry|
         chapter = self.chapters.find { |c| c.slug == entry[:slug] }
-        
+
         if chapter
           latest_chapter_at = [chapter.published_at || Time.at(0), latest_chapter_at].max
           chapter.set(**entry)
@@ -99,7 +130,7 @@ module FicTracker::Models
         end
         if to_remove.any?
           logger.debug "Removing chapter(s) #{to_remove.inspect} from story #{self}"
-          chapter_dataset.where(id: to_remove).delete
+          chapter_dataset.where(id: to_remove).destroy
         end
 
         update(updated_at: latest_chapter_at) if latest_chapter_at > Time.at(0) && (updated_at.nil? || latest_chapter_at >= updated_at)
@@ -109,16 +140,15 @@ module FicTracker::Models
     end
 
     def ensure_fully_loaded
-      ensure_chapters
       refresh_content
       refresh_metadata
+
+      # FIXME: Should check for a reasonable set of parameters - full load unless XX% (75%?) of chapters have content
+      backend.load_full_story(self) unless chapters && chapters.any? && chapters.all? { |c| c.content? && c.content_type? }
     end
 
     def ensure_chapters
-      # FIXME: Should check for a reasonable set of parameters - full load unless XX% (75%?) of chapters have content
+      ensure_fully_loaded
-      return if chapters && chapters.any? && chapters.all? { |c| c.content? && c.content_type? }
-
-      backend.load_full_story(self)
     end
 
     def backend
@@ -133,7 +163,7 @@ module FicTracker::Models
     end
 
     def etag
-      chapters.select { |c| c.etag }.last
+      chapters.select { |c| c.etag }.compact.last&.etag
     end
 
     def cache_key
@@ -153,25 +183,50 @@ module FicTracker::Models
     end
 
     def refresh_content
-      backend.find_chapters(self) if backend && needs_content_refresh?
+      refresh_content! if backend && needs_content_refresh?
-      chapters.each(&:refresh_content)
+      # chapters.each(&:refresh_content)
     end
 
     def refresh_content!
       backend.find_chapters(self)
-      chapters.each(&:refresh_content!)
+      # chapters.each(&:refresh_content!)
     end
 
     def needs_metadata_refresh?
+      return true if id && authors.empty?
+
       Time.now - (last_metadata_refresh || Time.at(0)) >= METADATA_REFRESH_INTERVAL
     end
 
+    def self.needing_metadata_refresh
+      where { Sequel.|(last_metadata_refresh.nil?, last_metadata_refresh < date.function(Time.now - METADATA_REFRESH_INTERVAL, 'localtime')) }
+    end
+
     def needs_content_refresh?
       Time.now - (last_content_refresh || Time.at(0)) >= (completed? ? METADATA_REFRESH_INTERVAL : CONTENT_REFRESH_INTERVAL)
     end
 
+    def self.needing_content_refresh
+      where { Sequel.|(last_content_refresh.nil?, last_content_refresh < date.function(Time.now - (completed? ? METADATA_REFRESH_INTERVAL : CONTENT_REFRESH_INTERVAL), 'localtime')) }
+    end
+
     def to_s
-      "#{name}, by #{author.nil? ? '<Unknown>' : author.to_s}"
+      author_names = self.authors.map(&:to_s)
+      if author_names.empty?
+        author_names = '<Unknown>'
+      elsif author_names.size == 1
+        author_names = author_names.first
+      else
+        author_names = author_names.reduce('') do |string, aut|
+          string = string.dup
+          string += 'and ' if aut != author_names.first && aut == author_names.last
+          string += aut
+          string += ', ' unless aut == author_names.last
+          string
+        end
+      end
+
+      "#{name}, by #{author_names}"
     end
 
     def uid

lib/fic_tracker/renderers.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module FicTracker::Renderers
+  autoload :Epub, 'fic_tracker/renderers/epub'
+  autoload :HTML, 'fic_tracker/renderers/html'
+  autoload :Markdown, 'fic_tracker/renderers/markdown'
+
+  def self.render(type, story, **attrs)
+    klass = case type
+            when :Markdown, :markdown, :md
+              Markdown
+            when :HTML, :html
+              HTML
+            when :Epub, :epub
+              Epub
+            end
+
+    stringio = nil
+    unless attrs[:io]
+      require 'stringio'
+      attrs[:io] = StringIO.new
+      stringio = true
+    end
+
+    result = klass.new(story, **attrs).render
+    return result unless stringio
+
+    attrs[:io].string
+  end
+end

lib/fic_tracker/renderers/epub.rb

@@ -69,6 +69,8 @@ module FicTracker::Renderers
     end
 
     def build_content_opf
+      require_relative '../util/time_extensions'
+
       Nokogiri::XML::Builder.new(encoding: 'UTF-8') do |xml|
         xml.package version: '2.0', xmlns: 'http://www.idpf.org/2007/opf', 'unique-identifier': 'story-url' do
           xml.metadata 'xmlns:dc': 'http://purl.org/dc/elements/1.1/', 'xmlns:opf': 'http://www.idpf.org/2007/opf' do
@@ -79,12 +81,15 @@ module FicTracker::Renderers
               require_relative '../converters/from_html'
               xml['dc'].description FicTracker::Converters::FromHTML.to_plain(@story.synopsis)
             end
-            xml['dc'].creator story.author&.to_s || '<Unknown>', 'opf:role': 'aut', 'opf:file-as': (story.author&.to_s || 'Unknown')
+            story.authors.each do |aut|
+              xml['dc'].creator aut.to_s, 'opf:role': 'aut', 'opf:file-as': aut.to_s
+            end
+            xml['dc'].creator '<Unknown>', 'opf:role': 'aut', 'opf:file-as': '<Unknown>' if story.authors.empty?
             xml['dc'].publisher story.backend.full_name
-            xml['dc'].date (story.published_at || Time.now).to_datetime
+            xml['dc'].date (story.published_at || story.updated_at || Time.now).to_datetime
             xml['dc'].relation story.backend.url
             xml['dc'].source story.url if story.url
-            story.tags.each do |tag|
+            story.tags&.each do |tag|
               xml['dc'].subject tag.to_s
             end
             xml.meta name: 'cover', content: 'coverImage' if cover
@@ -92,6 +97,7 @@ module FicTracker::Renderers
             xml.meta name: 'ft-backend', content: story.backend_name
             xml.meta name: 'ft-story', content: story.slug
             xml.meta name: 'ft-etag', content: story.etag
+            xml.meta name: 'ft-modified', content: (story.updated_at || story.published_at || Time.now).to_header
             xml.meta name: 'ft-complete', content: story.completed?
           end
 
@@ -117,7 +123,7 @@ module FicTracker::Renderers
     def build_toc_ncx
       Nokogiri::XML::Builder.new(encoding: 'UTF-8') do |xml|
         xml.doc.create_internal_subset(
-          'ncx', 
+          'ncx',
           '-//NISO//DTD ncx 2005-1//EN',
           'http://www.daisy.org/z3986/2005/ncx-2005-1.dtd'
         )

lib/fic_tracker/renderers/html.rb

@@ -44,11 +44,16 @@ module FicTracker::Renderers
 
         html.address do
           html << "By "
-          if story.author&.url
+          story.authors.each do |author|
-            html.a story.author.to_s, href: story.author.url
+            html << 'and ' if author != story.authors.first && author == story.authors.last
-          else
+            if author.url
-            html.i story.author&.to_s || '<Unknown>'
+              html.a author.to_s, href: author.url
+            else
+              html.i author.to_s
+            end
+            html << ', ' unless author == story.authors.last
           end
+          html.i '<Unknown>' if story.authors.empty?
         end
         html.a "On #{story.backend.full_name}.", href: story.url if story.url
 
@@ -64,7 +69,7 @@ module FicTracker::Renderers
         html.br
 
         html.dl do
-          story.tags.map { |td| FicTracker::Models::Light::Tag.load td }.sort_by(&:ordering).group_by(&:category).each do |category, tags|
+          story.tags&.map { |td| FicTracker::Models::Light::Tag.load td }.sort_by(&:ordering).group_by(&:category).each do |category, tags|
             html.dt category.to_s.capitalize
             html.dd do
               tags.each do |tag|
@@ -107,7 +112,7 @@ module FicTracker::Renderers
         raise "Unknown chapter content-type: #{chapter.content_type.inspect}"
       end
     end
-    
+
     private
 
     def logger
@@ -117,17 +122,23 @@ module FicTracker::Renderers
     def build_html(html)
       return yield if body_only
 
+      require_relative '../util/time_extensions'
+
       html.html(lang: story.language || 'en') do
         html.head do
           html.meta charset: 'utf-8'
-          html.meta viewport: 'width=device-width, initial-scale=1'
+          html.meta name: 'viewport', content: 'width=device-width, initial-scale=1'
-          html.meta name: 'author', content: story.author.to_s
+          story.authors.each do |aut|
+            html.meta name: 'author', content: aut.to_s
+          end
           html.meta name: 'generator', content: "FicTracker/#{FicTracker::VERSION}"
           html.meta name: 'keywords', content: story.tags.map(&:to_s).join(',')
 
           html.meta name: 'ft-backend', content: story.backend_name
           html.meta name: 'ft-story', content: story.slug
           html.meta name: 'ft-etag', content: story.etag
+          html.meta name: 'ft-modified', content: (story.updated_at || story.published_at || Time.now).to_header
+          html.meta name: 'ft-complete', content: story.completed?
 
           html.title story.to_s
         end

lib/fic_tracker/server.rb

@@ -5,6 +5,12 @@ require 'sinatra/base'
 module FicTracker
   # Web server for providing your fic tracking needs
   class Server < Sinatra::Base
+    def initialize(*)
+      @task_runner = Thread.new { background_tasks }
+
+      super
+    end
+
     configure :development do
       require 'sinatra/reloader'
       register Sinatra::Reloader
@@ -45,7 +51,13 @@ module FicTracker
     end
 
     get '/search/:backend', provides: :json do |_backend_name|
-      backend.get_search_info.to_json
+      search = backend.get_search_info
+      if params['tag'] || params['name']
+        raise "Missing params, must specify both tag and name" unless params['tag'] && params['name']
+        return search.class.find_tags(params['tag'], params['name']).to_json
+      end
+
+      search.to_info_json
     end
 
     post '/search/:backend', provides: :html do |_backend_name|
@@ -86,6 +98,72 @@ module FicTracker
       collection&.save_changes
     end
 
+    head '/story/:backend/*.*' do |_backend_name, slug, format|
+      mime = nil
+      case format
+      when 'epub', :epub
+        format = :epub
+        mime = 'application/epub+zip'
+      when 'html', :html
+        format = :html
+        mime = 'text/html'
+      when 'txt', :txt, 'md', :md
+        format = :markdown
+        mime = 'text/markdown'
+      else
+        halt 400, "Unknown format #{format}"
+      end
+
+      content_type mime
+      attachment "#{story.safe_name}.#{format}"
+
+      story = Models::Story.find(backend_name: backend.name, slug:)
+      if story
+        story.set(last_accessed: Time.now)
+
+        last_modified story.updated_at || story.published_at
+        etag story.etag
+      end
+    ensure
+      story&.save_changes
+    end
+
+    # rubocop:disable Metrics/BlockLength
+    get '/story/:backend/*.*' do |_backend_name, slug, format|
+      mime = nil
+      case format
+      when 'epub', :epub
+        format = :epub
+        mime = 'application/epub+zip'
+      when 'html', :html
+        format = :html
+        mime = 'text/html'
+      when 'txt', :txt, 'md', :md
+        format = :markdown
+        mime = 'text/markdown'
+      else
+        halt 400, "Unknown format #{format}"
+      end
+
+      story = Models::Story.find(backend_name: backend.name, slug:)
+      story ||= Models::Story.new(backend_name: backend.name, slug:)
+
+      story.ensure_fully_loaded unless story.chapters&.any?
+      story.set(last_accessed: Time.now)
+
+      content_type mime
+      attachment "#{story.safe_name}.#{format}"
+
+      last_modified story.updated_at || story.published_at
+      etag story.etag
+
+      story.ensure_fully_loaded
+      FicTracker::Renderers.render(format, story)
+    ensure
+      story&.save_changes
+    end
+    # rubocop:enable Metrics/BlockLength
+
     get '/story/:backend/:slug', provides: :html do |_backend_name, slug|
       story = Models::Story.find(backend_name: backend.name, slug:)
       story ||= Models::Story.new(backend_name: backend.name, slug:)
@@ -105,8 +183,7 @@ module FicTracker
       story = Models::Story.find(backend_name: backend.name, slug:)
       story ||= Models::Story.new(backend_name: backend.name, slug:)
 
-      story.refresh_content
+      story.ensure_fully_loaded
-      story.refresh_metadata
       story.set(last_accessed: Time.now)
 
       chapter = story.chapters[index.to_i]
@@ -119,44 +196,27 @@ module FicTracker
       story&.save_changes
     end
 
-    # rubocop:disable Metrics/BlockLength
+    private
-    get '/story/:backend/:slug.:format' do |_backend_name, slug, format|
+
-      render_klass = nil
+    def background_tasks
-      mime = nil
+      $stderr.puts "Starting background task loop"
-      case format
+      loop do
-      when 'epub', :epub
+        FicTracker::Models::Story.expire
-        render_klass = Renderers::Epub
+        FicTracker::Models::Author.expire
-        mime = 'application/epub+zip'
+
-      when 'html', :html
+        FicTracker::Models::Story.needing_content_refresh.each(&:refresh_content)
-        render_klass = Renderers::HTML
+        FicTracker::Models::Story.needing_metadata_refresh.each(&:refresh_metadata)
-        mime = 'text/html'
+        FicTracker::Models::Author.needing_metadata_refresh.each(&:refresh_metadata)
-      when 'txt', :txt, 'md', :md
+
-        render_klass = Renderers::Markdown
+        FicTracker.cache.expire
-        mime = 'text/markdown'
+      rescue StandardError => e
-      else
+        FicTracker.logger.error "Failed when running background tasks, #{e.class}: #{e}\n#{e.backtrace[-5,5].join("\n  ")}"
-        halt 400, "Unknown format #{format}"
+      ensure
+        iter += 1
+        sleep 30 * 60
       end
-
+    rescue StandardError => e
-      story = Models::Story.find(backend_name: backend.name, slug:)
+      $stderr.puts "Fatal background error: #{e.class}: #{e}"
-      story ||= Models::Story.new(backend_name: backend.name, slug:)
-
-      story.refresh_content
-      story.refresh_metadata
-      story.set(last_accessed: Time.now)
-
-      attachment "#{story.safe_name}.#{format}"
-      content_type mime
-
-      last_modified story.updated_at || story.published_at
-      etag story.etag
-
-      story.ensure_fully_loaded
-      stream do |out|
-        render_klass.new(story, io: out).render
-      end
-    ensure
-      story&.save_changes
     end
-    # rubocop:enable Metrics/BlockLength
   end
 end

lib/fic_tracker/tasks.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module FicTracker::Tasks
+  autoload :Cleanup, 'fic_tracker/tasks/cleanup'
+end

lib/fic_tracker/util/cache.rb

@@ -176,8 +176,8 @@ module FicTracker::Util
       if (flags & COMPRESSED_FLAG) == COMPRESSED_FLAG
         require 'zlib'
         data = Zlib::Inflate.inflate(data)
-        data.force_encoding('UTF-8')
       end
+      data.force_encoding('UTF-8')
 
       case (flags & ENCODING_FLAGS)
       when ENCODING_MARSHAL

lib/fic_tracker/util/cache/database.rb

@@ -7,8 +7,8 @@ module FicTracker::Util::CacheImpl
 
     def initialize(table: 'cache', **redis)
       @dataset = FicTracker.database[table.to_s.to_sym]
-      @dataset_expired = @dataset.exclude{ Sequel.|({ expire_at: nil }, Sequel::CURRENT_DATE < expire_at) }
       @dataset_live = @dataset.where{ Sequel.|({ expire_at: nil }, Sequel::CURRENT_DATE < expire_at) }
+      @dataset_expired = @dataset_live.invert
     end
 
     def expire
@@ -16,7 +16,7 @@ module FicTracker::Util::CacheImpl
     end
 
     def has?(key)
-      dataset_live.filter(key: key).any?
+      !dataset_live.filter(key: key).empty?
     end
 
     def get(key)

lib/fic_tracker/util/time_extensions.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class Time
+  def to_header
+    getgm.strftime("%a, %d %b %Y %T GMT")
+  end
+end