Initial commit

.github/workflows/main.yml

@@ -0,0 +1,27 @@
+name: Ruby
+
+on:
+  push:
+    branches:
+      - master
+
+  pull_request:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    name: Ruby ${{ matrix.ruby }}
+    strategy:
+      matrix:
+        ruby:
+          - '3.2.3'
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+        bundler-cache: true
+    - name: Run the default task
+      run: bundle exec rake

.gitignore

@@ -0,0 +1,12 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+/vendor/
+/Gemfile.lock
+/database.db
+/config.yml

.rubocop.yml

@@ -0,0 +1,13 @@
+AllCops:
+  TargetRubyVersion: 3.2
+
+Style/StringLiterals:
+  Enabled: true
+  # EnforcedStyle: single_quotes
+
+Style/StringLiteralsInInterpolation:
+  Enabled: true
+  # EnforcedStyle: single_quotes
+
+Layout/LineLength:
+  Max: 120

CHANGELOG.md

@@ -0,0 +1,5 @@
+## [Unreleased]
+
+## [0.1.0] - 2024-04-25
+
+- Initial release

Gemfile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in fic_tracker.gemspec
+gemspec
+
+gem "rake", "~> 13.0"
+
+gem "minitest", "~> 5.16"
+
+gem "rubocop", "~> 1.21"

LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2024 Alexander Olofsson
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

README.md

@@ -0,0 +1,16 @@
+# FicTracker
+
+A web application to handle tracking and reading live fiction, as it's updated.
+
+
+## Usage
+
+Instantiate config.yml and use rackup
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/ananace/ruby-fic_tracker.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

Rakefile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "minitest/test_task"
+
+Minitest::TestTask.create
+
+require "rubocop/rake_task"
+
+RuboCop::RakeTask.new
+
+task default: %i[test rubocop]

bin/fic_tracker

@@ -0,0 +1,73 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'fic_tracker'
+require 'optparse'
+require 'ostruct'
+
+options = OpenStruct.new
+OptParse.new do |opts|
+  opts.banner = 'Usage: fic_tracker [OPTIONS...]'
+  
+  opts.on '-b', '--backend=BACKEND', 'The backend to use' do |backend|
+    options.backend = backend
+  end
+
+  opts.on '-s', '--story=STORY', 'The story to download' do |story|
+    options.story = story
+  end
+
+  opts.on '-c', '--chapter=CHAPTER', 'The chapter to download' do |chapter|
+    options.chapter = chapter
+  end
+
+  opts.on '-f', '--format=FORMAT', 'The format to download (epub, html, markdown)' do |format|
+    options.format = format.to_sym
+  end
+
+  opts.on '-o', '--output=FILE', 'The resulting file to save the download into' do |output|
+    options.output = output
+  end
+
+  opts.separator ''
+
+  opts.on '-C', '--config=FILE', 'Specify a configuration file to read' do |config|
+    options.config = config
+  end
+
+  opts.on '-h', '--help', 'Show this text' do
+    puts opts
+    exit
+  end
+
+  opts.on '-q', '--quiet', 'Run quietly' do
+    options.log_level = :error
+  end
+
+  opts.on '-v', '--verbose', 'Run verbosely, can be specified twice for debug output' do
+    if %i[info debug].include?(options.log_level)
+      options.log_level = :debug
+    else
+      options.log_level = :info
+    end
+  end
+
+  opts.on '-V', '--version', 'Prints the version and exits' do
+    puts FicTracker::VERSION
+    exit
+  end
+end.parse!
+
+FicTracker.logger.level = options.log_level || :warn
+FicTracker.configure!
+
+backend = FicTracker::Backends.get(options.backend)
+story = FicTracker::Models::Story.find(backend_name: backend.name, slug: backend.parse_slug(options.story)) || FicTracker::Models::Story.new(backend: backend, slug: backend.parse_slug(options.story))
+story.ensure_fully_loaded
+data = nil
+
+options.output ||= "#{story.safe_name}.#{options.format}"
+FicTracker.logger.info "Saving to #{options.output}"
+File.open(options.output, 'w') { |f| FicTracker::Renderers.render(options.format, story, io: f) }
+
+story.save_changes

config.ru

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require 'fic_tracker'
+
+FicTracker.configure!
+
+map '/' ->() { run FicTracker::Server }

config.yml.example

@@ -0,0 +1,6 @@
+---
+database:
+  url: sqlite://database.db
+
+cache:
+  type: database

fic_tracker.gemspec

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require_relative "lib/fic_tracker/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "fic_tracker"
+  spec.version = FicTracker::VERSION
+  spec.authors = ["Alexander Olofsson"]
+  spec.email = ["ace@haxalot.com"]
+
+  spec.summary = 'A tracker and aggregator for live fiction'
+  spec.description = 'A web service that tracks and aggregates live fiction for use with e-readers or similar devices'
+  spec.homepage = 'https://github.com/ananace/ruby-fic_tracker'
+  spec.license = 'MIT'
+  spec.required_ruby_version = '>= 3.0.0'
+
+  spec.metadata["allowed_push_host"] = ''
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = spec.homepage
+  spec.metadata["changelog_uri"] = "#{spec.homepage}/blob/master/CHANGELOG.md"
+
+  spec.files = Dir['{lib,views,public}/**/*'] + %w[CHANGELOG.md LICENSE.txt README.md config.ru config.yml.example fic_tracker.gemspec]
+
+  spec.add_dependency 'haml'
+  spec.add_dependency 'kramdown'
+  spec.add_dependency 'logging'
+  spec.add_dependency 'nokogiri'
+  spec.add_dependency 'rackup'
+  spec.add_dependency 'ruby-vips'
+  spec.add_dependency 'rubyzip'
+  spec.add_dependency 'sequel'
+  spec.add_dependency 'sinatra'
+  spec.add_dependency 'sqlite3'
+end

lib/fic_tracker.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+require 'logging'
+
+require_relative 'fic_tracker/version'
+
+module FicTracker
+  class Error < StandardError; end
+
+  class ConfigError < Error; end
+
+  def self.logger
+    @logger ||= Logging.logger[self].tap do |logger|
+      logger.add_appenders ::Logging.appenders.stdout
+      logger.level = :warn
+    end
+  end
+
+  def self.configure!
+    Config.load!
+  end
+
+  def self.cache
+    configure! unless @cache
+    @cache
+  end
+
+  def self.cache=(cache)
+    @cache = cache
+  end
+
+  def self.database
+    configure! unless @database
+    @database
+  end
+
+  def self.database=(database)
+    @database = database
+  end
+
+  def self.debug!
+    logger.level = :debug
+  end
+
+  def self.logger=(global_logger)
+    @logger = global_logger
+    @global_logger = !global_logger.nil?
+  end
+
+  def self.global_logger?
+    @global_logger ||= false
+  end
+
+  module Models
+    def self.const_missing(const)
+      raise 'No database connected' unless FicTracker.database
+
+      model = const.to_s.downcase
+      require_relative "fic_tracker/models/#{model}"
+
+      mod = const_get(const) if const_defined? const
+      return mod if mod
+
+      raise "Model not found: #{const}"
+    end
+  end
+
+  module Tasks
+    autoload :Cleanup, 'fic_tracker/tasks/cleanup'
+  end
+
+  module Renderers
+    autoload :Epub, 'fic_tracker/renderers/epub'
+    autoload :HTML, 'fic_tracker/renderers/html'
+    autoload :Markdown, 'fic_tracker/renderers/markdown'
+
+    def self.render(type, story, **attrs)
+      klass = case type
+              when :Markdown, :markdown, :md
+                Markdown
+              when :HTML, :html
+                HTML
+              when :Epub, :epub
+                Epub
+              end
+ 
+      unless attrs[:io]
+        require 'stringio'
+        attrs[:io] = StringIO.new
+        attrs[:_stringio] = true
+      end
+
+      return klass.new(story, **attrs).render unless attrs[:stringio]
+
+      attrs[:io].string
+    end
+  end
+end
+
+require_relative 'fic_tracker/backend'
+require_relative 'fic_tracker/config'

lib/fic_tracker/backend.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module FicTracker
+  class Backend
+    class SearchInfo
+      attr_accessor :tag_categories
+    end
+
+    def self.backend_name
+      name.split('::')[-2]
+    end
+
+    def self.config_name
+      backend_name.downcase.to_sym
+    end
+
+    def name
+      self.class.backend_name
+    end
+
+    def full_name
+      self.class.name
+    end
+
+    def url; end
+
+    def load_author(author); end
+    def load_story(slug); end
+    def find_chapters(story); end
+    def load_full_story(story)
+      story = load_story(story)
+      find_chapters(story)
+      story.chapters.each { |c| c.refresh_content }
+      story
+    end
+    def load_chapter(slug, story); end
+
+    def get_search_info; end
+    def search(search_info); end
+
+    def parse_slug(slug)
+      slug
+    end
+
+    def cache_key
+      [ self.class.name.split('::').map(&:downcase)[-2, 1] ]
+    end
+
+    protected
+
+    def logger
+      Logging.logger[self]
+    end
+
+    def cache
+      FicTracker.cache
+    end
+  end
+
+  module Backends
+    def self.get(name)
+      const = name
+      const = const.to_s.to_sym unless const.is_a? Symbol
+
+      model = constants.find { |c| c == const }
+      return const_get(model).const_get(:Backend).new if model
+
+      model = load(name)
+      return model if model
+
+      raise "Unknown backend #{name}"
+    end
+
+    def self.const_missing(const)
+      mod = load(const)
+      return mod if mod
+
+      raise "Unknown backend #{const}"
+    end
+
+    def self.load(name)
+      model = name.to_s.downcase
+      require_relative "backends/#{model}/backend"
+
+      model = constants.find { |c| c.to_s.downcase == model }
+      return unless model
+
+      backend = const_get(model).const_get(:Backend)
+      backend.new(**FicTracker.config.dig(:backends, backend.config_name, default: {}))
+    rescue StandardError => e
+      Logging.logger[backend].error "Failed to load, #{e.class}: #{e}"
+
+      nil
+    end
+  end
+end

lib/fic_tracker/backends/ao3/backend.rb

@@ -0,0 +1,204 @@
+# frozen_string_literal: true
+
+require_relative 'client'
+
+module FicTracker::Backends::Ao3
+  class Backend < FicTracker::Backend
+    TAG_ORDERING = {
+      rating:   -3,
+      category: -2,
+      warning:  -1,
+    }.freeze
+  
+    def client
+      @client ||= Client.new
+    end
+
+    def full_name
+      'Archive of Our Own'
+    end
+
+    def url
+      Client::BASE_URL
+    end
+
+    def load_author(author)
+      author = FicTracker::Models::Author.new(slug: parse_slug(author), backend: self) unless author.is_a? FicTracker::Models::Author
+
+      logger.info "Loading author #{author.slug}"
+      doc = client.request("/users/#{author.slug}")
+
+      user = doc.at_css('#main .user')
+
+      url = user.at_css('.icon a')
+      url = URI.join(Client::BASE_URL, url[:href]) if url
+      image = user.at_css('.icon a img')
+      image = URI.join(Client::BASE_URL, image[:src]) if image
+
+      author.set(
+        name: user.at_css('h2.heading').text.strip,
+        url: url.to_s,
+        image: image.to_s,
+        last_metadata_refresh: Time.now
+      )
+    end
+
+    def load_story(story)
+      story = FicTracker::Models::Story.new(slug: parse_slug(story), backend: self) unless story.is_a? FicTracker::Models::Story
+
+      logger.info "Loading story #{story.slug}"
+      doc = client.request("/works/#{story.slug}")
+
+      attrs = extract_story doc
+
+      story.set(last_metadata_refresh: Time.now, **attrs)
+    end
+
+    def find_chapters(story)
+      story = FicTracker::Models::Story.new(slug: parse_slug(story), backend: self) unless story.is_a? FicTracker::Models::Story
+
+      logger.info "Loading chapters for #{story.slug}"
+      doc = client.request("/works/#{story.slug}/navigate")
+
+      chapters = doc.at_css('ol.chapter').css('li').map do |entry|
+        published_at = Time.parse(entry.at_css('span.datetime').text.strip)
+        link = entry.at_css('a')
+        index, *name = link.text.split('. ')
+        index = index.to_i
+        name = name.join('. ')
+        url = URI.join(Client::BASE_URL, link[:href])
+        slug = url.path.split('/').last
+
+        {
+          slug: slug,
+          index: index,
+          name: name,
+          url: url.to_s,
+          published_at: published_at,
+        }
+      end
+
+      story.set(
+        last_content_refresh: Time.now,
+      )
+      story.chapters = chapters
+
+      story
+    end
+
+    def load_full_story(story)
+      story = FicTracker::Models::Story.new(slug: parse_slug(story), backend: self) unless story.is_a? FicTracker::Models::Story
+
+      logger.info "Loading all chapters for #{story.slug}"
+      doc = client.request("/works/#{story.slug}", query: { view_full_work: true })
+
+      attrs = extract_story(doc)
+      chapters = doc.css('#chapters > div.chapter').map { |chapter| extract_chapter(chapter) }
+
+      story.set(
+        last_metadata_refresh: Time.now,
+        last_content_refresh: Time.now,
+        **attrs
+      )
+      story.chapters = chapters
+
+      story
+    end
+
+    def load_chapter(chapter, story)
+      story = load_story(story) unless story.is_a? FicTracker::Models::Story
+      chapter = FicTracker::Models::Chapter.new(slug: parse_slug(chapter), story: story) unless chapter.is_a? FicTracker::Models::Chapter
+
+      logger.info "Loading chapter #{chapter.slug} for #{story.slug}"
+      doc = client.request("/works/#{story.slug}/chapters/#{chapter.slug}")
+
+      attrs = extract_chapter(doc.at_css('#chapters > div.chapter'))
+
+      chapter.set(**attrs)
+    end
+
+    def parse_slug(slug)
+      return URI(slug).path.split('/').last if slug.is_a?(String) && slug.start_with?('http')
+
+      slug.to_s
+    end
+
+    private
+
+    def extract_story(doc)
+      url = URI.join(Client::BASE_URL, doc.at_css('li.share a')[:href].delete_suffix('/share'))
+      meta = doc.at_css('#main dl.meta')
+      preface = doc.at_css('#workskin .preface')
+
+      name = preface.at_css('.title').text.strip
+      synopsis = preface.at_css('.summary blockquote').children.to_xml
+      language = meta.at_css('dd.language')['lang']
+      author = preface.at_css('a[rel="author"]')[:href].split('/')[2]
+
+      tags = meta.css('dd.tags').map do |tagblock|
+        category = tagblock[:class].split.first.to_sym
+
+        tagblock.css('a.tag').map do |a|
+          next if ['Creator Chose Not To Use Archive Warnings', 'No Archive Warnings Apply'].include? a.text.strip
+
+          {
+            name: a.text.strip,
+            category: category,
+            important: %i[rating warning category].include?(category) ? true : nil,
+            ordering: TAG_ORDERING[category],
+          }.compact
+        end.compact
+      end.flatten
+
+      chapters = meta.at_css('dl.stats dd.chapters').text.strip.split('/')
+      words = meta.at_css('dl.stats dd.words').text.strip.tr(',', '').to_i
+      published_at = meta.at_css('dl.stats dd.published')&.text&.strip
+      published_at = Time.parse(published_at) if published_at
+      updated_at = meta.at_css('dl.stats dd.status')&.text&.strip
+      updated_at = Time.parse(updated_at) if updated_at
+      
+      {
+        name: name,
+        author: author,
+        synopsis: synopsis,
+        url: url.to_s,
+        language: language,
+        chapter_count: chapters.first.to_i, 
+        word_count: words, 
+        completed: chapters.first == chapters.last,
+        published_at: published_at,
+        updated_at: updated_at,
+        tags: FicTracker::Models::Light::Tag.load(tags),
+      }
+    end
+
+    def extract_chapter(chapter_doc)
+      link = chapter_doc.at_css('h3.title a')
+      url = URI.join(Client::BASE_URL, link[:href])
+      slug = url.path.split('/').last
+      index = link.text.strip.split.last.to_i
+
+      html = chapter_doc.at_css('div[role="article"]').children[2..].map(&:to_xml).join("\n")
+
+      title = chapter_doc.at_css('h3.title')
+      if title
+        title_base = title.at_css('a').text.strip
+        title_extra = title.text.strip.delete_prefix("#{title_base}:").strip
+
+        title = title_extra.empty? ? title_base : title_extra
+      end
+
+      { 
+        index: index,
+        slug: slug,
+
+        name: title,
+        url: url.to_s,
+        last_refresh: Time.now,
+
+        content: html,
+        content_type: 'text/html',
+      }
+    end
+  end
+end

lib/fic_tracker/backends/ao3/client.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+require 'cgi'
+require 'net/http'
+require 'nokogiri'
+require 'json'
+
+module FicTracker::Backends::Ao3
+  class Client
+    BASE_URL = 'https://archiveofourown.org'
+
+    def url
+      URI(BASE_URL)
+    end
+
+    def request(path, type: :html, body: nil, query: nil, method: :get, redirect: true)
+      uri = URI.join(url, path)
+      uri.query = URI.encode_www_form(query) if query
+
+      req = Net::HTTP.const_get(method.to_s.capitalize.to_sym).new uri.request_uri
+      req['User-Agent'] = "FicTracker/#{FicTracker::VERSION}"
+      case type
+      when :html, :xml
+        req['Accept'] = 'text/html,application/xhtml+xml,application/xml'
+      when :json
+        req['Accept'] = 'application/json'
+      end
+
+      if body
+        req.body = body
+        req.body = req.body.to_json unless req.body.is_a? String
+        req['Content-Type'] = 'application/json'
+      end
+
+      resp = nil
+      http.start do
+        loop do
+          debug_http(req)
+          resp = http.request req
+          debug_http(resp)
+          case resp
+          when Net::HTTPRedirection
+            req.path.replace resp['location']
+          when Net::HTTPTooManyRequests
+            wait_time = 10
+            if resp['retry-after']
+              after = resp['retry-after']
+              wait_time = after =~ /\A[0-9]+\Z/ ? after.to_i : Time.parse(after)
+            end
+            wait_time = Time.now + wait_time if wait_time.is_a? Numeric
+
+            logger.info "Rate limited, waiting until #{wait_time} before retrying"
+            sleep Time.now - wait_time
+          else
+            break
+          end
+        end
+      end
+
+      resp.value
+      return if resp.body.empty?
+
+      case type
+      when :html
+        Nokogiri::HTML4.parse(resp.body)
+      when :xml
+        Nokogiri::XML.parse(resp.body)
+      when :json
+        JSON.parse(resp.body, symbolize_names: true)
+      when :raw
+        resp.body
+      end
+    end
+
+    private
+
+    def logger
+      Logging.logger[self]
+    end
+
+    def http
+      @http ||= Net::HTTP.new(url.host, url.port).tap { |http| http.use_ssl = url.scheme == 'https' }
+    end
+
+    def debug_http(object)
+      return unless logger.debug?
+
+      dir = '>'
+      if object.is_a?(Net::HTTPRequest)
+        dir = '<'
+
+        logger.debug "#{dir} #{object.method} #{object.path}"
+      else
+        logger.debug "#{dir} #{object.code} #{object.message}"
+      end
+
+      object.each_header { |h, v| logger.debug "#{dir} #{h}: #{v}" }
+    end
+  end
+end

lib/fic_tracker/backends/fimfiction/backend.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require_relative 'fimfiction/client'
+
+module FicTracker::Backends::FIMFiction
+  class Backend < FicTracker::Backend
+    def client
+      @client ||= Client.new
+    end
+
+    def full_name
+      'FIM Fiction'
+    end
+
+    def url
+      'https://www.fimfiction.net'
+    end
+
+    def load_author(author)
+      author = FicTracker::Models::Author.new(slug: author, backend: self) unless author.is_a? FicTracker::Models::Author
+
+      logger.info "Loading author #{author.slug}"
+    end
+    def load_story(slug)
+      story = FicTracker::Models::Story.new(slug: story, backend: self) unless story.is_a? FicTracker::Models::Story
+
+      logger.info "Loading story #{story.slug}"
+    end
+    def find_chapters(story)
+      story = FicTracker::Models::Story.new(slug: story, backend: self) unless story.is_a? FicTracker::Models::Story
+
+      logger.info "Loading chapters for #{story.slug}"
+    end
+    def load_chapter(chapter, story)
+      story = FicTracker::Models::Story.new(slug: story, backend: self) unless story.is_a? FicTracker::Models::Story
+      chapter = FicTracker::Models::Chapter.new(slug: chapter, story: story) unless chapter.is_a? FicTracker::Models::Chapter
+
+      logger.info "Loading chapter #{chapter.slug} for #{story.slug}"
+    end
+
+    def get_search_info; end
+    def search(search_info); end
+  end
+end

lib/fic_tracker/backends/fimfiction/client.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+require 'cgi'
+require 'json'
+require 'net/http'
+require 'uri'
+
+module FicTracker::Backends::FIMFiction
+  class Client
+    BASE_URL = URI('https://www.fimfiction.net/api/v2').freeze
+
+    attr_accessor :access_token, :url
+    attr_reader :tags
+
+    def initialize
+      @access_token = nil
+      @url = BASE_URL
+
+      @tags = Tags.new(self)
+    end
+
+    def request(path, body: nil, query: nil, method: :get)
+      uri = URI.join(BASE_URL, path)
+      uri.query = URI.encode_www_form(query) if query
+
+      req = Net::HTTP.const_get(method.to_s.capitalize.to_sym).new 
+      req['Accept'] = 'application/vnd.api+json'
+      req['Authorization'] = "Bearer #{access_token}" if access_token
+      req['User-Agent'] = "FicTracker/#{FicTracker::VERSION}"
+
+      if body
+        req.body = body
+        req.body = req.body.to_json unless req.body.is_a? String
+        req['Content-Type'] = 'application/json'
+      end
+
+      resp = nil
+      http.start do
+        loop do
+          debug_http(req)
+          resp = http.request req
+          debug_http(resp)
+          case resp
+          when Net::HTTPRedirection
+            req.path.replace resp['location']
+          when Net::HTTPTooManyRequests
+            wait_time = 10
+            if resp['retry-after']
+              after = resp['retry-after']
+              wait_time = after =~ /\A[0-9]+\Z/ ? after.to_i : Time.parse(after)
+            end
+            wait_time = Time.now + wait_time if wait_time.is_a? Numeric
+
+            logger.info "Rate limited, waiting until #{wait_time} before retrying"
+            sleep Time.now - wait_time
+          else
+            break
+          end
+        end
+      end
+
+      resp.value
+      return if resp.body.empty?
+
+      return JSON.parse(resp.body, symbolize_names: true)
+    end
+
+    def retrieve_token(id, secret)
+      data = request('/token', body: { client_id: id, client_secret: secret, grant_type: :client_credentials }, method: :post)
+
+      @access_token = data[:access_token]
+    end
+
+    private
+
+    def logger
+      Logging.logger[self]
+    end
+
+    def http
+      @http ||= Net::HTTP.new(url.host, url.port).tap { |http| http.use_ssl = url.scheme == 'https' }
+    end
+
+    def debug_http(object)
+      return unless logger.debug?
+
+      dir = '>'
+      if object.is_a?(Net::HTTPRequest)
+        dir = '<'
+
+        logger.debug "#{dir} #{object.method} #{object.path}"
+      else
+        logger.debug "#{dir} #{object.code} #{object.message}"
+      end
+
+      object.each_header { |h, v| logger.debug "#{dir} #{h}: #{v}" }
+    end
+  end
+end

lib/fic_tracker/config.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require 'psych'
+
+require_relative 'util/cache'
+require_relative 'util/database'
+
+module FicTracker::Config
+  class << self
+    def load!(config_file: 'config.yml')
+      @config_file = config_file
+      @config = {}
+
+      load_internal
+
+      FicTracker.database = FicTracker::Util::Database.connect(
+        dig(:database, :url, default: :memory),
+        migrate: dig(:database, :migrate, default: true),
+        options: dig(:database, :options, default: {})
+      )
+      FicTracker.cache = FicTracker::Util::Cache.create(
+        type: dig(:cache, :type, default: :none),
+        compress: dig(:cache, :compress, default: nil),
+        encoding: dig(:cache, :encoding, default: nil),
+        options: dig(:cache, :options, default: {})
+      )
+
+      Sequel::Model.db = FicTracker.database
+    end
+
+    def [](arg)
+      @config[arg]
+    end
+
+    def dig(*args, default: nil)
+      envvar = (%i[ft] + args.map { |arg| arg.to_s }).join('_').upcase
+      return ENV[envvar] if ENV[envvar]
+
+      ret = @config.dig(*args)
+      return ret if ret
+      return default unless default.nil?
+      return yield if block_given?
+
+      nil
+    end
+
+    private
+
+    def load_internal
+      @config_file = ENV['FT_CONFIG_FILE'] if ENV['FT_CONFIG_FILE']
+      begin
+        @config = Psych.load(File.read(@config_file)).deep_transform_keys(&:to_sym)
+      rescue
+        @config = {}
+      end
+      @config[:database] ||= {}
+      @config[:database][:config] ||= {}
+      @config[:cache] ||= {}
+
+      if ENV['FT_CACHE_REDIS_URL']
+        @config[:cache][:type] = :redis
+        @config[:cache][:url] = ENV['FT_CACHE_REDIS_URL']
+      end
+    end
+  end
+end

lib/fic_tracker/converters/from_html.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+require 'kramdown'
+
+module FicTracker::Converters
+  class FromHTML
+    def self.to_plain(content)
+      require_relative 'to_plain'
+
+      Kramdown::Document.new(content, input: 'html').to_plain.strip
+    end
+
+    def self.to_md(content)
+      require_relative 'to_simplemark'
+
+      Kramdown::Document.new(content, input: 'html').to_simplemark.strip
+    end
+
+    def self.to_safe_html(content)
+      require_relative 'to_safe_html'
+
+      Kramdown::Document.new(content, input: 'html').to_safe_html
+    end
+  end
+end

lib/fic_tracker/converters/from_markdown.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require 'kramdown'
+
+module FicAggregator::Converters
+  class FromMarkdown
+    def self.to_plain(content)
+      require_relative 'to_plain'
+
+      Kramdown::Document.new(content, input: 'markdown').to_plain.strip
+    end
+
+    def self.to_html(content)
+      Kramdown::Document.new(content, input: 'markdown').to_html.strip
+    end
+
+    def self.to_safe_html(content)
+      Kramdown::Document.new(content, input: 'markdown').to_safe_html
+    end
+  end
+end

lib/fic_tracker/converters/to_plain.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require_relative 'to_simplemark'
+
+module Kramdown::Converter
+  # Markdown with some slight simplification to ease reading
+  class Plain < Simplemark
+    private
+
+    def convert_header(el, depth = 1)
+      depth = 7 - depth
+      "#{'=' * depth} #{inner(el)} #{'=' * depth}\n\n"
+    end
+
+    def convert_strong(el)
+      "*#{inner(el)}*"
+    end
+
+    def convert_em(el)
+      "_#{inner(el)}_"
+    end
+
+    def convert_a(el)
+      content = inner(el)
+      [content, "(#{el.attr['href']})"].reject { |k| k.nil? || k.empty? }.join ' '
+    end
+    
+    def convert_blockquote(el)
+      "   #{inner(el).split("\n").join("\n    ")}"
+    end
+  end
+end

lib/fic_tracker/converters/to_safe_html.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+require 'kramdown'
+require 'logging'
+
+module Kramdown::Converter
+  # HTML converter that strips out elements not relevant for e-readers
+  class SafeHtml < Base
+    def initialize(root, options)
+      super
+  
+      @options[:remove_block_html_tags] = true
+      @options[:remove_span_html_tags] = false
+      @options[:template] = 'string://<%= Html.convert(@body).first %>'
+    end
+
+    SUPERFLUOUS_TAGS = %w[align class justify]
+
+    def convert(el)
+      real_el, el = el, el.value if el.type == :footnote
+
+      # Strip out unnecessary HTML tags
+      SUPERFLUOUS_TAGS.each { |tag| el.attr.delete tag if el.attr.key? tag }
+
+      children = el.children.dup
+      index = 0
+
+      while index < children.length
+        if %i[img xml_pi].include?(children[index].type) ||
+            (children[index].type == :html_element && %w[style script title].include?(children[index].value))
+          children[index..index] = []
+        elsif children[index].type == :html_element && (
+            (@options[:remove_block_html_tags] && children[index].options[:category] == :block) ||
+            (@options[:remove_span_html_tags] && children[index].options[:category] == :span)
+          ) && children[index].value != 'div'
+          children[index..index] = children[index].children
+        else
+          convert(children[index])
+          index += 1
+        end
+      end
+      el.children = children
+
+      real_el || el
+    end
+  end
+end

lib/fic_tracker/converters/to_simplemark.rb

@@ -0,0 +1,176 @@
+# frozen_string_literal: true
+
+require 'kramdown'
+require 'logging'
+
+module Kramdown::Converter
+  # Simplified commonmark converter
+  class Simplemark < Base
+    def convert(el, **_)
+      meth = :"convert_#{el.type}"
+      meth = :"convert_html_#{el.value}" if el.type == :html_element
+
+      if self.class.private_method_defined? meth
+        res = send(meth, el)
+      else
+        fallback = :convert_blank
+        fallback = :convert_html_element if el.type == :html_element
+
+        logger.debug "Missing #{meth.inspect} function, using #{fallback}"
+        res = send(fallback, el)
+      end
+      res = res.dup if res.frozen?
+      res
+    end
+
+    private
+
+    def logger
+      Logging.logger[self]
+    end
+
+    class UlBullet
+      def next
+        '-'
+      end
+    end
+
+    class OlBullet
+      def initialize
+        @at = 1
+      end
+
+      def next
+        "#{@at}."
+      ensure
+        @at = @at.succ!
+      end
+    end
+
+    def inner(el)
+      el.children.map { |el| convert(el) }.join
+    end
+    alias convert_root inner
+
+    def convert_blank(el)
+      ''
+    end
+    alias convert_comment convert_blank
+
+    def convert_p(el)
+      "#{inner(el)}\n\n"
+    end
+
+    def convert_text(el)
+      el.value
+    end
+    alias convert_abbreviation convert_text
+
+    def convert_strong(el)
+      "__#{inner(el).gsub('__', '\_\_')}__"
+    end
+    alias convert_html_b convert_strong
+    alias convert_html_strong convert_strong
+
+    def convert_em(el)
+      "_#{inner(el).gsub('_', '\_')}_"
+    end
+    alias convert_html_i convert_em
+    alias convert_html_em convert_em
+
+    def convert_del(el)
+      "~#{inner(el)}~"
+    end
+    alias convert_html_del convert_del
+
+    def convert_codespan(el)
+      "`#{el.value}`"
+    end
+    alias convert_html_code convert_codespan
+    
+    def convert_blockquote(el)
+      "> #{inner(el).split("\n").join("\n> ")}\n\n"
+    end
+
+    def convert_codeblock(el)
+      "```\n#{el.value}```\n\n"
+    end
+
+    def convert_a(el)
+      content = inner(el)
+      "[#{content}](#{el.attr['href']})"
+    end
+    alias convert_html_a convert_a
+
+    def convert_header(el, depth = 1)
+      "#{'#' * depth} #{inner(el)} #{'#' * depth}\n\n"
+    end
+    def convert_html_h1(el); convert_header(el, 1) end
+    def convert_html_h2(el); convert_header(el, 2) end
+    def convert_html_h3(el); convert_header(el, 3) end
+    def convert_html_h4(el); convert_header(el, 4) end
+    def convert_html_h5(el); convert_header(el, 5) end
+    def convert_html_h6(el); convert_header(el, 6) end
+    
+    def convert_br(el)
+      "  \n"
+    end
+
+    def convert_hr(el)
+      "#{'-' * 16}\n\n"
+    end
+
+    def convert_ul(el)
+      @bullet = UlBullet.new
+      "#{inner(el)}\n"
+    ensure
+      @bullet = nil
+    end
+    alias convert_html_ul convert_ul
+
+    def convert_ol(el)
+      @bullet = OlBullet.new
+      "#{inner(el)}\n"
+    ensure
+      @bullet = nil
+    end
+    alias convert_html_ol convert_ol
+
+    def convert_li(el)
+      "  #{@bullet.next} #{inner(el).strip}\n"
+    end
+    alias convert_html_li convert_li
+
+    def convert_smart_quote(el)
+      SMART_QUOTES.fetch(el.value)
+    end
+
+    def convert_typographic_sym(el)
+      TYPOGRAPHIC_SYMBOLS.fetch(el.value)
+    end
+
+    def convert_entity(el)
+      el.value.char
+    end
+
+    alias convert_html_title convert_blank
+
+    # Strip out unknown HTML elements, only keep content
+    alias convert_html_element inner
+
+    SMART_QUOTES = {
+      lsquo: "‘",
+      rsquo: "’",
+      ldquo: "“",
+      rdquo: "”" }.freeze
+
+    TYPOGRAPHIC_SYMBOLS = {
+      mdash: "—",
+      ndash: "–",
+      hellip: "...",
+      laquo_space: "« ",
+      raquo_space: " »",
+      laquo: "«",
+      raquo: "»" }.freeze
+  end
+end

lib/fic_tracker/migrations/001_create_base.rb

@@ -0,0 +1,180 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  change do
+    create_table(:meta) do
+      String :key, null: false, primary_key: true
+      String :value, null: true
+    end
+
+    create_table(:cache) do
+      String :key, null: false, primary_key: true
+      File :value, null: true
+      DateTime :expire_at, null: true, default: nil
+    end
+
+    create_table(:authors) do
+      primary_key :id
+      String :slug, null: false
+      String :backend_name, null: false
+      index %i[slug backend_name], unique: true
+
+      String :name, null: false
+
+      String :url, null: true, default: nil
+      String :image, null: true, default: nil
+
+      DateTime :last_metadata_refresh, null: true, default: nil
+
+      # JSON
+      String :data, null: false, text: true, default: '{}'
+    end
+
+    create_table(:chapters) do
+      primary_key :id
+
+      foreign_key :story_id, :stories, on_delete: :cascade, on_update: :cascade
+      Integer :index, null: false
+      String :slug, null: false
+      index %i[story_id index], unique: true
+      index %i[story_id slug], unique: true
+
+      String :etag, null: true, default: nil
+      String :name, null: true, default: nil
+
+      String :url, null: true, default: nil
+
+      DateTime :published_at, null: true, default: nil
+      DateTime :updated_at, null: true, default: nil
+      DateTime :last_refresh, null: true, default: nil
+
+      # JSON
+      String :data, null: false, text: true, default: '{}'
+    end
+
+    create_table(:collections) do
+      primary_key :id
+      String :slug, null: false
+      String :backend_name, null: false
+      index %i[slug backend_name], unique: true
+
+      String :type, null: false
+      String :name, null: true, default: nil
+      String :url, null: true, default: nil
+
+      Integer :story_count, null: false, default: 0
+      Boolean :completed, null: false, default: false
+
+      DateTime :last_refresh, null: true, default: nil
+
+      # JSON
+      String :data, null: false, text: true, default: '{}'
+    end
+
+    create_table(:collection_stories) do
+      primary_key :id
+
+      foreign_key :collection_id, :collections, on_delete: :cascade, on_update: :cascade
+      foreign_key :story_id, :stories, on_delete: :cascade, on_update: :cascade
+      Integer :index, null: true, default: nil
+    end
+
+    create_table(:stories) do
+      primary_key :id
+      String :slug, null: false
+      String :backend_name, null: false
+      index %i[slug backend_name], unique: true
+
+      foreign_key :author_id, :authors, on_delete: :cascade, on_update: :cascade
+
+      String :name, null: false
+      String :synopsis, null: false, text: true
+      String :language, null: true, default: 'en'
+
+      String :url, null: true, default: nil
+      String :image, null: true, default: nil
+
+      Integer :chapter_count, null: false, default: 0
+      Integer :word_count, null: false, default: 0
+      Boolean :completed, null: false, default: false
+
+      DateTime :published_at, null: false, default: Sequel::CURRENT_TIMESTAMP
+      DateTime :updated_at, null: false, default: Sequel::CURRENT_TIMESTAMP
+
+      DateTime :last_metadata_refresh, null: true, default: nil
+      DateTime :last_content_refresh, null: true, default: nil
+      DateTime :last_accessed, null: false, default: Sequel::CURRENT_TIMESTAMP
+
+      # JSON
+      String :tags, null: false, text: true, default: '[]'
+      String :data, null: false, text: true, default: '{}'
+    end
+
+    if false
+      create_table(:users) do
+        primary_key :id
+
+        String :username, null: false
+        String :email, null: true, default: nil
+        String :mxid, null: true, default: nil
+
+        # JSON
+        String :data, null: false, text: true, default: '{}'
+      end
+
+      create_table(:user_backend_auth) do
+        foreign_key :user_id, :users, on_delete: :cascade, on_update: :cascade
+        String :backend_name, null: false
+
+        # JSON
+        String :authdata, null: false, text: true, default: '{}'
+      end
+
+      create_table(:user_history) do
+        primary_key :id
+
+        foreign_key :user_id, :users, on_delete: :cascade, on_update: :cascade
+        String :slug, null: false
+        String :backend_name, null: false
+        index %i[user_id slug backend_name], unique: true
+
+        Integer :chapter_index, null: false, default: 0
+        # XPATH selector for furthest page element that was reached
+        String :chapter_cursor, null: true, default: nil
+
+        # JSON
+        String :data, null: true, text: true, default: '{}'
+      end
+
+      create_table(:user_sessions) do
+        primary_key :id
+
+        foreign_key :user_id, :users, on_delete: :cascade, on_update: :cascade
+        String :auth_token, null: false
+        String :refresh_token, null: false
+
+        DateTime :session_lifetime, null: true, default: nil
+        DateTime :last_sync, null: false, default: Sequel::CURRENT_TIMESTAMP
+
+        # JSON
+        String :data, null: false, text: true, default: '{}'
+      end
+
+      create_table(:user_tracked) do
+        primary_key :id
+
+        foreign_key :user_id, :users, on_delete: :cascade, on_update: :cascade
+        foreign_key :story_id, :stories, on_delete: :cascade, on_update: :cascade
+        index %i[user_id story_id], unique: true
+
+        Integer :chapter_index, null: false, default: 0
+        # XPATH selector for furthest page element that was reached
+        String :chapter_cursor, null: true, default: nil
+
+        # JSON
+        String :data, null: false, text: true, default: '{}'
+      end
+    end
+  end
+end
+

lib/fic_tracker/models/author.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module FicTracker::Models
+  class Author < Sequel::Model
+    # 1/week
+    METADATA_REFRESH_INTERVAL = 7 * 24 * 60 * 60
+
+    plugin :serialization, :json, :data
+
+    one_to_many :stories
+
+    def before_create
+      backend.load_author(self) if last_metadata_refresh.nil?
+
+      super
+    end
+
+    def backend
+      return unless backend_name
+
+      @backend ||= FicTracker::Backends.get(backend_name)
+    end
+
+    def backend=(backend)
+      @backend = backend
+      self.backend_name = backend.name
+    end
+
+    def refresh_metadata
+      return unless backend && needs_metadata_refresh?
+
+      refresh_metadata!
+    end
+
+    def refresh_metadata!
+      backend.load_author(self)
+    end
+
+    def needs_metadata_refresh?
+      Time.now - (last_metadata_refresh || Time.at(0)) >= METADATA_REFRESH_INTERVAL
+    end
+
+    def to_s
+      name || slug
+    end
+  end
+end
+

lib/fic_tracker/models/chapter.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+require 'digest'
+
+module FicTracker::Models
+  class Chapter < Sequel::Model
+    # 3/day
+    CONTENT_REFRESH_INTERVAL = 12 * 60 * 60
+    CONTENT_CACHE_TIME = 7 * 24 * 60 * 60
+
+    plugin :serialization, :json, :data
+
+    many_to_one :story
+
+    def to_s
+      return "Chapter #{index}" unless name
+
+      "Chapter #{index}: #{name}"
+    end
+
+    def backend
+      story&.backend
+    end
+
+    def content?
+      return true if @content
+      return false unless cache_key
+
+      key = cache_key + [:content]
+      FicTracker.cache.has?(key)
+    end
+
+    def content
+      return @content if @content
+      return unless cache_key
+
+      key = cache_key + [:content]
+      refresh_content! unless FicTracker.cache.has?(key)
+
+      @content ||= FicTracker.cache.get(key)
+    end
+
+    def content=(content)
+      if cache_key
+        key = cache_key + [:content]
+        FicTracker.cache.set(key, content, CONTENT_CACHE_TIME)
+      end
+      @content = content
+      etag = Digest::SHA1.hexdigest(content)
+    end
+
+    def content_type?
+      return true if @content_type
+      return false unless cache_key
+
+      key = cache_key + [:content]
+      FicTracker.cache.has?(key)
+    end
+
+    def content_type
+      return @content_type if @content_type
+      return unless cache_key
+
+      key = cache_key + [:content_type]
+      refresh_content! unless FicTracker.cache.has?(key)
+
+      @content_type ||= FicTracker.cache.get(key)
+    end
+
+    def content_type=(type)
+      if cache_key
+        key = cache_key + [:content_type]
+        FicTracker.cache.set(key, type, CONTENT_CACHE_TIME)
+      end
+      @content_type = type
+    end
+
+    def cache_key
+      return unless backend
+
+      backend.cache_key + [:c, slug]
+    end
+
+    def refresh_content
+      return unless backend && needs_content_refresh?
+
+      refresh_content!
+    end
+
+    def refresh_content!
+      backend.load_chapter(self, story)
+    end
+
+    def needs_content_refresh?
+      Time.now - (last_refresh || Time.at(0)) >= CONTENT_REFRESH_INTERVAL
+    end
+  end
+end

lib/fic_tracker/models/light/search_info.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+
+module FicTracker::Models::Light
+  class SearchInfo
+    def with_tag(tag)
+      self
+    end
+
+    def without_tag(tag)
+      self
+    end
+
+    def with_words(count)
+      self
+    end
+
+    class << self
+      protected
+      
+      def tag_category(category, **params)
+      end
+
+      def word_limits(*limits)
+      end
+
+      private
+    end
+  end
+end

lib/fic_tracker/models/light/tag.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module FicTracker::Models::Light
+  class Tag
+    def self.load(data)
+      return data if data.is_a?(Tag) || (data.is_a?(Array) && data.all? { |d| d.is_a? Tag })
+      return data.map { |d| load(d) } if data.is_a?(Array) && data.all? { |d| d.is_a? Hash }
+      return new(**data) if data.is_a? Hash
+
+      JSON.parse(data, symbolize_names: true).map { |obj| new(**obj) }
+    end
+
+    def self.store(data)
+      JSON.generate(data)
+    end
+
+    attr_accessor :id, :name, :category
+    attr_writer :ordering, :important
+
+    def initialize(id: nil, name: nil, category: nil, important: nil, ordering: nil, **params)
+      @id = id
+      @name = name
+      @category = category&.to_sym
+      @important = important
+      @ordering = ordering
+    end
+
+    def important?
+      @important
+    end
+
+    def ordering
+      @ordering || 0
+    end
+
+    def update(**attrs)
+      attrs.each do |k, v|
+        next unless respond_to(:"#{k}=")
+
+        send ":#{k}=", v
+      end
+    end
+
+    def to_s
+      "#{category}: #{name}"
+    end
+
+    def to_json(*opts)
+      {
+        id: @id,
+        name: @name,
+        category: @category,
+        important: @important,
+        ordering: @ordering,
+      }.compact.to_json(*opts)
+    end
+  end
+end

lib/fic_tracker/models/story.rb

@@ -0,0 +1,187 @@
+# frozen_string_literal: true
+
+require_relative 'light/tag'
+
+module FicTracker::Models
+  class Story < Sequel::Model
+    # 1/week
+    METADATA_REFRESH_INTERVAL = 7 * 24 * 60 * 60
+    # 3/day
+    CONTENT_REFRESH_INTERVAL = 12 * 60 * 60
+
+    many_to_one :author
+    one_to_many :chapters, order: :index
+    many_to_many :collection, join_table: :collection_stories
+
+    plugin :serialization, [Light::Tag.method(:store), Light::Tag.method(:load)], :tags
+    plugin :serialization, :json, :data
+
+    # Defer creation of author/chapters until story requiring them is to be saved
+    def before_create
+      if @author
+        @author.save unless @author.id
+        self.author_id = @author.id
+      end
+
+      super
+    end
+
+    def after_create
+      return if [@author, @chapters].all?(&:nil?)
+
+      self.author = @author if @author
+      @author = nil
+
+      if @chapters
+        latest_chapter_at = self.updated_at || self.published_at || Time.at(0)
+        @chapters&.each do |chapter|
+          latest_chapter_at = [chapter.published_at || Time.at(0), latest_chapter_at].max
+          add_chapter chapter
+        end
+        update(updated_at: latest_chapter_at) if latest_chapter_at > Time.at(0) && (updated_at.nil? || latest_chapter_at >= updated_at)
+      end
+      @chapters = nil
+    end
+
+    def completed?
+      completed
+    end
+
+    # Support attaching author to a not-yet-saved story
+    def author
+      @author || super
+    end
+
+    def author=(author_name)
+      author = author_name if author_name.is_a?(FicTracker::Models::Author)
+      author ||= Author.find(backend_name: backend.name, slug: author_name)
+      author ||= backend.load_author(author_name) if id
+      author ||= Author.new(backend: backend, slug: author_name)
+      
+      if id
+        @author = nil
+        author.save unless author.id
+        super(author)
+      else
+        @author = author
+      end
+    end
+
+    # Support attaching chapters to a not-yet-saved story
+    def chapters
+      @chapters || super
+    end
+    
+    def chapters=(entries)
+      latest_chapter_at = self.published_at || Time.at(0)
+
+      to_add = []
+      to_remove = self.chapters.map(&:id)
+
+      entries.each do |entry|
+        chapter = self.chapters.find { |c| c.slug == entry[:slug] }
+        
+        if chapter
+          latest_chapter_at = [chapter.published_at || Time.at(0), latest_chapter_at].max
+          chapter.set(**entry)
+        else
+          chapter = FicTracker::Models::Chapter.new(**entry)
+          to_add << chapter
+        end
+        to_remove.delete chapter.id
+      end
+
+      if id
+        @chapters = nil
+        to_add.each do |entry|
+          logger.debug "Adding new chapter #{entry.inspect} to story #{self}"
+          add_chapter entry
+        end
+        if to_remove.any?
+          logger.debug "Removing chapter(s) #{to_remove.inspect} from story #{self}"
+          chapter_dataset.where(id: to_remove).delete
+        end
+
+        update(updated_at: latest_chapter_at) if latest_chapter_at > Time.at(0) && (updated_at.nil? || latest_chapter_at >= updated_at)
+      else
+        @chapters = (@chapters || []) + to_add - to_remove
+      end
+    end
+
+    def ensure_fully_loaded
+      ensure_chapters
+      refresh_content
+      refresh_metadata
+    end
+
+    def ensure_chapters
+      # FIXME: Should check for a reasonable set of parameters - full load unless XX% (75%?) of chapters have content
+      return if chapters && chapters.any? && chapters.all? { |c| c.content? && c.content_type? }
+
+      backend.load_full_story(self)
+    end
+
+    def backend
+      return unless backend_name
+
+      @backend ||= FicTracker::Backends.get(backend_name)
+    end
+
+    def backend=(backend)
+      @backend = backend
+      self.backend_name = backend.name
+    end
+
+    def etag
+      chapters.select { |c| c.etag }.last
+    end
+
+    def cache_key
+      backend.cache_key + [:s, slug]
+    end
+
+    def safe_name
+      [backend_name, slug, name].join('_').downcase.gsub(/[^a-z0-9\-_]/, '_').gsub(/__+/, '_')
+    end
+
+    def refresh_metadata
+      refresh_metadata! if backend && needs_metadata_refresh?
+    end
+
+    def refresh_metadata!
+      backend.load_story(self)
+    end
+
+    def refresh_content
+      backend.find_chapters(self) if backend && needs_content_refresh?
+      chapters.each(&:refresh_content)
+    end
+
+    def refresh_content!
+      backend.find_chapters(self)
+      chapters.each(&:refresh_content!)
+    end
+
+    def needs_metadata_refresh?
+      Time.now - (last_metadata_refresh || Time.at(0)) >= METADATA_REFRESH_INTERVAL
+    end
+
+    def needs_content_refresh?
+      Time.now - (last_content_refresh || Time.at(0)) >= (completed? ? METADATA_REFRESH_INTERVAL : CONTENT_REFRESH_INTERVAL)
+    end
+
+    def to_s
+      "#{name}, by #{author.nil? ? '<Unknown>' : author.to_s}"
+    end
+
+    def uid
+      Digest::SHA1.hexdigest("#{backend}/#{slug}")
+    end
+
+    private
+
+    def logger
+      Logging.logger[self]
+    end
+  end
+end

lib/fic_tracker/models/user.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module FicTracker::Models
+  class User < Sequel::Model
+    plugin :serialization, :json, :data
+
+    one_to_many :sessions
+    one_to_many :tracked
+    one_to_many :history
+
+  end
+end

lib/fic_tracker/renderers/epub.rb

@@ -0,0 +1,200 @@
+# frozen_string_literal: true
+
+module FicTracker::Renderers
+  class Epub
+    attr_accessor :cover, :preface
+    attr_reader :story, :html
+
+    def initialize(story, io:, cover: true, preface: true)
+      @story = story
+      @io = io
+
+      @cover = cover && story.image
+      @preface = preface
+
+      @html = HTML.new(story, io: nil, body_only: true)
+      @html.cover = 'images/cover.jpg' if @cover
+    end
+
+    def render
+      logger.info "Rendering epub for #{story}"
+
+      require 'zip'
+      require 'zip/filesystem'
+
+      Zip::File.open_buffer(@io, create: true, compression_level: Zlib::BEST_COMPRESSION) do |zip|
+        zip.dir.mkdir('META-INF')
+        zip.file.open('META-INF/container.xml', +'w') { |f| f.write build_container_xml }
+
+        if cover
+          zip.dir.mkdir('images')
+          zip.file.open('images/cover.jpg', +'w') { |f| f.write build_cover_image }
+        end
+
+        zip.file.open('mimetype', +'w') { |f| f.write 'application/epub+zip' }
+        zip.file.open('content.opf', +'w') { |f| f.write build_content_opf }
+        zip.file.open('toc.ncx', +'w') { |f| f.write build_toc_ncx }
+
+        zip.file.open('preface.xhtml', +'w') { |f| f.write build_preface_xhtml } if preface
+
+        story.chapters.each do |chapter|
+          zip.file.open("chapter#{chapter.index}.xhtml", +'w') { |f| f.write(build_chapter_xhtml(chapter)) }
+        end
+      end
+    end
+
+    private
+
+    def logger
+      Logging.logger[self]
+    end
+
+    def build_cover_image
+      require 'vips'
+
+      buf = Net::HTTP.get story.image
+      im = Vips::Image.thumbnail_buffer buf, 800, height: 1280, size: :down
+
+      im.write_to_buffer '.jpg', strip: true
+    end
+
+    def build_container_xml
+      Nokogiri::XML::Builder.new(encoding: 'UTF-8') do |xml|
+        xml.container version: '1.0', xmlns: 'urn:oasis:names:tc:opendocument:xmlns:container' do
+          xml.rootfiles do
+            xml.rootfile 'full-path': 'content.opf', 'media-type': 'application/oebps-package+xml'
+          end
+        end
+      end.to_xml(indent: 0)
+    end
+
+    def build_content_opf
+      Nokogiri::XML::Builder.new(encoding: 'UTF-8') do |xml|
+        xml.package version: '2.0', xmlns: 'http://www.idpf.org/2007/opf', 'unique-identifier': 'story-url' do
+          xml.metadata 'xmlns:dc': 'http://purl.org/dc/elements/1.1/', 'xmlns:opf': 'http://www.idpf.org/2007/opf' do
+            xml['dc'].title story.name
+            xml['dc'].language story.language || 'en'
+            xml['dc'].identifier story.url || "#{story.backend}/#{story.slug}", id: 'story-url', 'opf:scheme': 'url'
+            if story.synopsis
+              require_relative '../converters/from_html'
+              xml['dc'].description FicTracker::Converters::FromHTML.to_plain(@story.synopsis)
+            end
+            xml['dc'].creator story.author&.to_s || '<Unknown>', 'opf:role': 'aut', 'opf:file-as': (story.author&.to_s || 'Unknown')
+            xml['dc'].publisher story.backend.full_name
+            xml['dc'].date (story.published_at || Time.now).to_datetime
+            xml['dc'].relation story.backend.url
+            xml['dc'].source story.url if story.url
+            story.tags.each do |tag|
+              xml['dc'].subject tag.to_s
+            end
+            xml.meta name: 'cover', content: 'coverImage' if cover
+
+            xml.meta name: 'ft-backend', content: story.backend_name
+            xml.meta name: 'ft-story', content: story.slug
+            xml.meta name: 'ft-etag', content: story.etag
+            xml.meta name: 'ft-complete', content: story.completed?
+          end
+
+          xml.manifest do
+            xml.item id: 'coverImage', properties: 'cover-image', href: 'images/cover.jpg', 'media-type': 'image/jpeg' if cover
+            xml.item id: 'preface', href: 'preface.xhtml', 'media-type': 'application/xhtml+xml' if preface
+            story.chapters.each do |chapter|
+              xml.item id: "chapter#{chapter.index}", href: "chapter#{chapter.index}.xhtml", 'media-type': 'application/xhtml+xml'
+            end
+            xml.item id: 'ncx', href: 'toc.ncx', 'media-type': 'application/x-dtbncx+xml'
+          end
+
+          xml.spine(toc: 'ncx') do
+            xml.itemref idref: 'preface', linear: 'no' if preface
+            story.chapters.each do |chapter|
+              xml.itemref idref: "chapter#{chapter.index}"
+            end
+          end
+        end
+      end.to_xml(indent: 0)
+    end
+
+    def build_toc_ncx
+      Nokogiri::XML::Builder.new(encoding: 'UTF-8') do |xml|
+        xml.doc.create_internal_subset(
+          'ncx', 
+          '-//NISO//DTD ncx 2005-1//EN',
+          'http://www.daisy.org/z3986/2005/ncx-2005-1.dtd'
+        )
+        xml.ncx xmlns: 'http://www.daisy.org/z3986/2005/ncx/', version: '2005-1', 'xml:lang': story.language || 'en' do
+          xml.head do
+            xml.meta name: 'dtb:uid', content: story.url || "#{story.backend}/#{story.slug}"
+            xml.meta name: 'dtb:depth', content: 2
+            xml.meta name: 'dtb:generator', content: "FicTracker/#{FicTracker::VERSION}"
+            xml.meta name: 'dtb:totalPageCount', content: 0
+            xml.meta name: 'dtb:maxPageNumber', content: 0
+          end
+
+          xml.docTitle do
+            xml.text story.name
+          end
+          xml.docAuthor do
+            xml.text story.author&.to_s || '<Unknown>'
+          end
+
+          xml.navMap do
+            if preface
+              xml.navPoint id: 'chapter0', class: 'chapter', playOrder: '0' do
+                xml.navLabel do
+                  xml.text 'Preface'
+                end
+                xml.content 'preface.xhtml'
+              end
+            end
+            story.chapters.each do |chapter|
+              xml.navPoint id: "chapter#{chapter.index}", class: 'chapter', playOrder: chapter.index do
+                xml.navLabel do
+                  xml.text chapter.name
+                end
+                xml.content "chapter#{chapter.index}.xhtml"
+              end
+            end
+          end
+        end
+      end.to_xml(indent: 0)
+    end
+
+    def build_preface_xhtml
+      Nokogiri::XML::Builder.new(encoding: 'UTF-8') do |xml|
+        xml.doc.create_internal_subset(
+          'html',
+          '-//W3C//DTD XHTML 1.1//EN',
+          'http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd'
+        )
+        xml.html xmlns: 'http://www.w3.org/1999/xhtml', 'xml:lang': story.language || 'en' do
+          xml.head do
+            xml.meta 'http-equiv': 'Content-Type', content: 'application/xhtml+xml; charset=utf-8'
+            xml.title story.to_s
+          end
+          xml.body do
+            @html.build_preface(xml)
+          end
+        end
+      end.to_xml(indent: 0)
+    end
+
+    def build_chapter_xhtml(chapter)
+      Nokogiri::XML::Builder.new(encoding: 'UTF-8') do |xml|
+        xml.doc.create_internal_subset(
+          'html',
+          '-//W3C//DTD XHTML 1.1//EN',
+          'http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd'
+        )
+        xml.html xmlns: 'http://www.w3.org/1999/xhtml', 'xml:lang': story.language || 'en' do
+          xml.head do
+            xml.meta 'http-equiv': 'Content-Type', content: 'application/xhtml+xml; charset=utf-8'
+            xml.title chapter.to_s
+          end
+          xml.body do
+            @html.build_chapter(xml, chapter)
+          end
+        end
+      end.to_xml(indent: 0)
+    end
+  end
+end

lib/fic_tracker/renderers/html.rb

@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+module FicTracker::Renderers
+  class HTML
+    attr_accessor :body_only, :cover, :preface
+    attr_reader :story
+
+    def initialize(story, io:, cover: false, preface: true, body_only: false)
+      @story = story
+      @io = io
+
+      @cover = story.image if cover
+      @preface = preface
+      @body_only = body_only
+    end
+
+    def render
+      logger.info "Rendering html for #{story}"
+
+      doc = Nokogiri::HTML5::Document.new
+      Nokogiri::XML::Builder.with(doc) do |html|
+        build_html(html) do
+          html.article class: "story" do
+            if preface
+              build_preface(html)
+              html.hr
+            end
+
+            story.chapters.each do |chapter|
+              html.article class: 'chapter' do
+                build_chapter(html, chapter)
+              end
+              html.hr unless chapter == story.chapters.last
+            end
+          end
+        end
+      end
+      @io.puts doc.to_html
+    end
+
+    def build_preface(html)
+      html.article class: 'preface' do
+        html.h1 story.name
+
+        html.address do
+          html << "By "
+          if story.author&.url
+            html.a story.author.to_s, href: story.author.url
+          else
+            html.i story.author&.to_s || '<Unknown>'
+          end
+        end
+        html.a "On #{story.backend.full_name}.", href: story.url if story.url
+
+        html.span do
+          html << 'Published '
+          html.time story.published_at.to_date, datetime: story.published_at.iso8601
+          if story.updated_at && story.updated_at != story.published_at
+            html << ", #{story.completed? ? 'completed' : 'updated'} "
+            html.time story.updated_at.to_date, datetime: story.updated_at.iso8601
+          end
+          html << '.'
+        end
+        html.br
+
+        html.dl do
+          story.tags.map { |td| FicTracker::Models::Light::Tag.load td }.sort_by(&:ordering).group_by(&:category).each do |category, tags|
+            html.dt category.to_s.capitalize
+            html.dd do
+              tags.each do |tag|
+                if tag.important?
+                  html.b tag.name
+                else
+                  html << tag.name
+                end
+                html << ', ' unless tag == tags.last
+              end
+            end
+          end
+        end
+
+        html.article class: 'synopsis' do
+          html << story.synopsis
+        end
+
+        html.img alt: 'Cover image', src: cover if cover
+      end
+    end
+
+    def build_chapter(html, chapter)
+      html.h2 do
+        html.a chapter, href: chapter.url
+      end
+
+      case chapter.content_type
+      when 'text/html'
+        require_relative '../converters/from_html'
+        html << FicTracker::Converters::FromHTML.to_safe_html(chapter.content)
+      when 'text/markdown'
+        require_relative '../converters/from_markdown'
+        html << FicTracker::Converters::FromMarkdown.to_safe_html(chapter.content)
+      when 'text/plain'
+        chapter.content.split("\n\n").each do |para|
+          html.p para
+        end
+      else
+        raise "Unknown chapter content-type: #{chapter.content_type.inspect}"
+      end
+    end
+    
+    private
+
+    def logger
+      Logging.logger[self]
+    end
+
+    def build_html(html)
+      return yield if body_only
+
+      html.html(lang: story.language || 'en') do
+        html.head do
+          html.meta charset: 'utf-8'
+          html.meta viewport: 'width=device-width, initial-scale=1'
+          html.meta name: 'author', content: story.author.to_s
+          html.meta name: 'generator', content: "FicTracker/#{FicTracker::VERSION}"
+          html.meta name: 'keywords', content: story.tags.map(&:to_s).join(',')
+
+          html.meta name: 'ft-backend', content: story.backend_name
+          html.meta name: 'ft-story', content: story.slug
+          html.meta name: 'ft-etag', content: story.etag
+
+          html.title story.to_s
+        end
+        html.body do
+          yield
+        end
+      end
+    end
+  end
+end

lib/fic_tracker/renderers/markdown.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module FicTracker::Renderers
+  class Markdown
+    attr_accessor :cover, :preface
+    attr_reader :story
+
+    def initialize(story, io:, cover: false, preface: true)
+      @story = story
+      @io = io
+
+      @cover = cover && story.image
+      @preface = preface
+    end
+
+    def render
+      logger.info "Rendering markdown for #{story}"
+
+      @io.puts build_preface, nil if preface
+
+      story.chapters.each do |chapter|
+        @io.puts build_chapter(chapter), nil
+      end
+
+      @io.puts build_chapter_references
+    end
+
+    private
+
+    def logger
+      Logging.logger[self]
+    end
+
+    def build_preface
+      require_relative '../converters/from_html'
+      <<~EOF
+      #{story.name}
+      #{'=' * story.name.size}
+
+      By __*[#{story.author}][aut]*__#{'  '}
+      Published #{story.published_at.to_date}#{(story.updated_at && story.updated_at != story.published_at) ? ", updated #{story.updated_at.to_date}" : ''}.#{'  '}
+      [The original work.](#{story.url})
+      #{cover ? "![Cover image](#{story.image})\n" : ''}
+      #{FicTracker::Converters::FromHTML.to_md(story.synopsis)}
+      EOF
+    end
+
+    def build_chapter(chapter)
+      head = "[#{chapter}][ch#{chapter.index}]\n#{'-' * (chapter.to_s.size + 6 + chapter.index.to_s.size)}\n\n"
+
+      head + case chapter.content_type
+             when 'text/html'
+               require_relative '../converters/from_html'
+               FicTracker::Converters::FromHTML.to_md(chapter.content)
+             when 'text/markdown'
+               chapter.content
+             when 'text/plain'
+               chapter.content
+             end
+    end
+
+    def build_chapter_references
+      refs = []
+      refs << "[aut]: #{story.author.url}" if preface
+      refs += story.chapters.map { |c| "[ch#{c.index}]: #{c.url}" }
+      refs.join "\n"
+    end
+  end
+end

lib/fic_tracker/server.rb

@@ -0,0 +1,162 @@
+# frozen_string_literal: true
+
+require 'sinatra/base'
+
+module FicTracker
+  # Web server for providing your fic tracking needs
+  class Server < Sinatra::Base
+    configure :development do
+      require 'sinatra/reloader'
+      register Sinatra::Reloader
+    end
+
+    configure do
+      root = File.join(__dir__, '../..')
+
+      set :views, File.join(root, 'views')
+      set :public_folder, File.join(root, 'public')
+      enable :logging
+    end
+
+    helpers do
+      def backend
+        return env[:ft_backend] if env[:ft_backend]
+        return halt 400, 'No backend provided' unless params['backend']
+
+        env[:ft_backend] = Backends.get params['backend']
+        return halt 400, "Unable to find backend #{params['backend']}" unless env[:ft_backend]
+
+        env[:ft_backend]
+      end
+    end
+
+    before do
+      expires 3600, :public, :must_revalidate if request.request_method == 'GET'
+    end
+
+    get '/', provides: :html do
+      haml :index, format: :html5
+    end
+
+    get '/search/:backend', provides: :html do |_backend_name|
+      search = backend.get_search_info
+
+      haml :search, format: :html5, locals: { search:, backend: }
+    end
+
+    get '/search/:backend', provides: :json do |_backend_name|
+      backend.get_search_info.to_json
+    end
+
+    post '/search/:backend', provides: :html do |_backend_name|
+      request.body.rewind  # in case someone already read it
+      data = JSON.parse request.body.read, symbolize_names: true
+
+      results = backend.sarch(data)
+
+      haml :search_result, format: :html5, locals: { results:, backend: }
+    end
+
+    post '/search/:backend', provides: :json do |_backend_name|
+      request.body.rewind  # in case someone already read it
+      data = JSON.parse request.body.read, symbolize_names: true
+
+      results = backend.sarch(data)
+
+      results.to_json
+    end
+
+    get '/author/:backend/:slug', provides: :html do |_backend_name, slug|
+      author = Models::Author.find(backend_name: backend.name, slug:)
+      author ||= Models::Author.new(backend_name: backend.name, slug:)
+      author.refresh_metadata
+
+      haml :author, format: :html5, locals: { author:, backend: }
+    ensure
+      author&.save_changes
+    end
+
+    get '/collection/:backend/:slug', provides: :html do |_backend_name, slug|
+      collection = Models::Collection.find(backend_name: backend.name, slug:)
+      collection ||= Models::Collection.new(backend_name: backend.name, slug:)
+      collection.refresh_metadata
+
+      haml :collection, format: :html5, locals: { collection:, backend: }
+    ensure
+      collection&.save_changes
+    end
+
+    get '/story/:backend/:slug', provides: :html do |_backend_name, slug|
+      story = Models::Story.find(backend_name: backend.name, slug:)
+      story ||= Models::Story.new(backend_name: backend.name, slug:)
+
+      story.ensure_fully_loaded
+      story.set(last_accessed: Time.now)
+
+      last_modified story.updated_at || story.published_at
+      etag story.etag
+
+      haml :story, format: :html5, locals: { story:, backend: }
+    ensure
+      story&.save_changes
+    end
+
+    get '/story/:backend/:slug/:index', provides: :html do |_backend_name, slug, index|
+      story = Models::Story.find(backend_name: backend.name, slug:)
+      story ||= Models::Story.new(backend_name: backend.name, slug:)
+
+      story.refresh_content
+      story.refresh_metadata
+      story.set(last_accessed: Time.now)
+
+      chapter = story.chapters[index.to_i]
+
+      last_modified chapter.published_at
+      etag chapter.etag
+
+      haml :chapter, format: :html5, locals: { chapter:, story:, backend: }
+    ensure
+      story&.save_changes
+    end
+
+    # rubocop:disable Metrics/BlockLength
+    get '/story/:backend/:slug.:format' do |_backend_name, slug, format|
+      render_klass = nil
+      mime = nil
+      case format
+      when 'epub', :epub
+        render_klass = Renderers::Epub
+        mime = 'application/epub+zip'
+      when 'html', :html
+        render_klass = Renderers::HTML
+        mime = 'text/html'
+      when 'txt', :txt, 'md', :md
+        render_klass = Renderers::Markdown
+        mime = 'text/markdown'
+      else
+        halt 400, "Unknown format #{format}"
+      end
+
+      story = Models::Story.find(backend_name: backend.name, slug:)
+      story ||= Models::Story.new(backend_name: backend.name, slug:)
+
+      story.refresh_content
+      story.refresh_metadata
+      story.set(last_accessed: Time.now)
+
+      attachment "#{story.safe_name}.#{format}"
+      content_type mime
+
+      last_modified story.updated_at || story.published_at
+      etag story.etag
+
+      story.ensure_fully_loaded
+      stream do |out|
+        render_klass.new(story, io: out).render
+      end
+    ensure
+      story&.save_changes
+    end
+    # rubocop:enable Metrics/BlockLength
+  end
+end

lib/fic_tracker/tasks/cleanup.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module FicTracker::Tasks
+  module Cleanup
+    # Remove stories that haven't been accessed in a month
+    DELETE_UNACCESSED_STORIES = 30 * 24 * 60 * 60
+
+    def self.run
+      delete_unaccessed_stories
+      delete_unnecessary_authors
+      FicTracker.cache.expire
+    end
+
+    class <<self
+      private
+
+      def delete_unaccessed_stories
+        FicTracker::Models::Story.where{last_accessed + DELETE_UNACCESSED_STORIES < Sequel::CURRENT_DATE}.destroy
+      end
+
+      def delete_unnecessary_authors
+        FicTracker::Models::Author.exclude(author_id: FicTracker::Models::Story.select(:author_id)).destroy
+      end
+    end
+  end
+end

lib/fic_tracker/util/cache.rb

@@ -0,0 +1,196 @@
+# frozen_string_literal: true
+
+begin
+  require 'cbor'
+rescue LoadError
+  require 'json'
+end
+
+module FicTracker::Util
+  class Cache
+    include Enumerable
+
+    def self.create(type: :none, compress: nil, encoding: nil, options: {})
+      require_relative 'cache/base'
+      cache = nil
+      case type
+      when :memory, 'memory'
+        require_relative 'cache/memory'
+        cache = CacheImpl::Memory.new(**options)
+      when :database, 'database'
+        require_relative 'cache/database'
+        cache = CacheImpl::Database.new(**options)
+      when :file, 'file'
+        require_relative 'cache/file'
+        cache = CacheImpl::File.new(**options)
+      when :redis, 'redis'
+        require_relative 'cache/redis'
+        cache = CacheImpl::Redis.new(**options)
+      when :none, 'none', nil
+        require_relative 'cache/dummy'
+        cache = CacheImpl::Dummy.new
+      else
+        raise ConfigError, "Unknown cache type #{type.inspect}" unless type.respond_to?(:get) && type.respond_to?(:set)
+
+        cache = type
+      end
+
+      new(cache, compress: compress, encoding: encoding)
+    end
+
+    attr_accessor :compress, :encoding
+
+    def initialize(backend, compress: nil, encoding: nil)
+      @backend = backend
+
+      # Always compress >1.5KB unless otherwise specified
+      @compress = compress || 1500
+      @compress = (@compress..) if @compress.is_a? Numeric
+      @encoding = encoding
+    end
+
+    # Expire any unwanted entries from the cache
+    def expire
+      @backend.expire if @backend.respond_to? :expire
+    end
+
+    # Check if the cache contains a given key which has not expired
+    def has?(key)
+      key = expand_key(key)
+      return @backend.has?(key) if @backend.respond_to? :has?
+
+      !@backend.get(key).nil?
+    end
+    # Get the value of a key, will return nil if the key is expired
+    def get(key)
+      key = expand_key(key)
+      decode_data @backend.get(key)
+    end
+    # Set the value for a given key, with an optional expiry
+    def set(key, value, expire = nil)
+      key = expand_key(key)
+      @backend.set(key, encode_data(value), expire)
+    end
+    # Get the value for a given key, or set it as the result of the block with an optional expiry
+    def get_or_set(key, expire = nil, &block)
+      key = expand_key(key)
+      return decode_data(@backend.get_or_set(key, expire) { encode_data(block.call) }) if @backend.respond_to? :get_or_set
+      return get(key) if has?(key)
+
+      value = block.call
+
+      @backend.set(key, encode_data(value), expire)
+
+      value
+    end
+    # Clear the value for a given key
+    def delete(key)
+      key = expand_key(key)
+      @backend.delete key if @backend.respond_to? :delete
+    end
+    # Clear the entire cache
+    def clear
+      @backend.clear if @backend.respond_to? :clear
+    end
+
+    private
+
+    def logger
+      Logging.logger[self]
+    end
+
+    attr_accessor :encoder, :compress
+
+    ENCODING_FLAGS = 1 << 0 | 1 << 1 | 1 << 2
+    COMPRESSED_FLAG = 1 << 3
+
+    ENCODING_NONE = 0
+    ENCODING_MARSHAL = 1
+    ENCODING_JSON = 2
+    ENCODING_CBOR = 3
+    ENCODING_CUSTOM = 4
+
+    def expand_key(key)
+      return key.join '/' if key.is_a? Array
+
+      key
+    end
+
+    # Check if a piece of data is a plain-old-object - i.e. simple data
+    def is_pod?(data)
+      return true if data.nil? || data == true || data == false || data.is_a?(String) || data.is_a?(Numeric)
+      return true if data.is_a?(Array) && data.all? { |k| is_pod?(k) }
+      return true if data.is_a?(Hash && data.all? { |k, v| k.is_a?(Symbol) && is_pod?(v) })
+
+      false
+    end
+
+    def best_encoder?(data)
+      pod_encoder = :none if data.is_a?(String)
+      pod_encoder ||= :cbor if Object.const_defined?(:CBOR)
+      pod_encoder ||= :json if Object.const_defined?(:JSON)
+      pod_encoder ||= :marshal
+      return pod_encoder if is_pod?(data)
+
+      nil
+    end
+
+    def encode_data(data, encode: nil)
+      encode ||= @encoder
+      encode ||= best_encoder?(data)
+
+      flags = 0
+      case encode
+      when :none
+        data = data
+        flags |= ENCODING_NONE
+      when :marshal
+        data = Marshal.dump(data)
+        flags |= ENCODING_MARSHAL
+      when :json
+        data = data.to_json
+        flags |= ENCODING_JSON
+      when :cbor
+        data = data.to_cbor
+        flags |= ENCODING_CBOR
+      else
+        raise "Unknown encoder #{(encode || @encoder).inspect}" unless encode.nil? && @encoder.respond_to?(:dump)
+
+        @encode.dump(data)
+        flags |= ENCODING_CUSTOM
+      end
+
+      if @compress && @compress.include?(data.bytesize)
+        require 'zlib'
+        data = Zlib::Deflate.deflate(data, Zlib::BEST_COMPRESSION)
+        flags |= COMPRESSED_FLAG
+      end
+
+      data.dup.prepend([flags].pack('C'))
+    end
+
+    def decode_data(data)
+      flags = data[0].unpack('C').first
+      data = data[1..]
+
+      if (flags & COMPRESSED_FLAG) == COMPRESSED_FLAG
+        require 'zlib'
+        data = Zlib::Inflate.inflate(data)
+        data.force_encoding('UTF-8')
+      end
+
+      case (flags & ENCODING_FLAGS)
+      when ENCODING_MARSHAL
+        Marshal.load(data)
+      when ENCODING_JSON
+        JSON.parse(data, symbolize_names: true)
+      when ENCODING_CBOR
+        CBOR.decode(data)
+      when ENCODING_CUSTOM
+        @encode.load(data)
+      else
+        data
+      end
+    end
+  end
+end

lib/fic_tracker/util/cache/base.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module FicTracker::Util::CacheImpl
+  class Base
+    def has?(key); end
+    def get(key); end
+    def set(key, value, ttl); end
+    def delete(key); end
+  end
+end
+

lib/fic_tracker/util/cache/database.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module FicTracker::Util::CacheImpl
+  # A cache using live redis data
+  class Database < Base
+    attr_reader :dataset, :dataset_expired, :dataset_live
+
+    def initialize(table: 'cache', **redis)
+      @dataset = FicTracker.database[table.to_s.to_sym]
+      @dataset_expired = @dataset.exclude{ Sequel.|({ expire_at: nil }, Sequel::CURRENT_DATE < expire_at) }
+      @dataset_live = @dataset.where{ Sequel.|({ expire_at: nil }, Sequel::CURRENT_DATE < expire_at) }
+    end
+
+    def expire
+      dataset_expired.delete
+    end
+
+    def has?(key)
+      dataset_live.filter(key: key).any?
+    end
+
+    def get(key)
+      blob = dataset_live.filter(key: key).first&.[](:value)
+      return unless blob
+
+      blob.to_s
+    end
+
+    def set(key, value, expire = nil)
+      expire = expire >= 0 ? Time.now + expire : nil if expire.is_a?(Numeric)
+      expire = nil if expire.is_a?(Numeric)
+
+      blob = Sequel.blob(value)
+      dataset
+        .insert_conflict(target: :key, update: { value: blob, expire_at: expire })
+        .insert(key: key, value: blob, expire_at: expire)
+
+      value
+    end
+
+    def delete(key)
+      dataset.where(key: key).delete
+      nil
+    end
+
+    def clear
+      dataset.delete
+    end
+
+    private
+
+    def namespace_key(key)
+      "#{namespace}-#{key}"
+    end
+
+    def unnamespace_key(key)
+      key.delete_prefix("#{@namespace}-")
+    end
+  end
+end

lib/fic_tracker/util/cache/dummy.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module FicTracker::Util::CacheImpl
+  # A dummy cache that doesn't store any data
+  class Dummy < Base
+  end
+end

lib/fic_tracker/util/cache/file.rb

@@ -0,0 +1,99 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'tmpdir'
+
+module FicTracker::Util::CacheImpl
+  # A filesystem-backed in-memory cache
+  class File < Base
+    def initialize(dir: nil)
+      @dir = dir || Dir.mktmpdir('ficagg')
+
+      @internal = Memory.new
+    end
+
+    def expire
+      @internal.lock do
+        Dir[File.join(@dir, '**')].each_child do |entry|
+          next unless entry.end_with?('.meta')
+
+          meta = JSON.parse(File.read(entry), symbolize_names: true)
+          unset meta[:key] unless meta_valid?(meta)
+        end
+      end
+    end
+
+    def has?(key)
+      internal_has = @internal.has?(key)
+      return internal_has if internal_has
+
+      File.exist?(key_path(key)) && key_valid?(key)
+    end
+
+    def get(key)
+      # TODO: Extract the metadata ttl into the memory cache
+      @internal.get_or_set(key, expiry: Time.at(0)) do
+        File.read(key_path(key)) if has?(key)
+      end
+    end
+
+    def set(key, value, expiry = nil)
+      expiry = expiry >= 0 ? Time.now + expiry : nil if expiry.is_a?(Numeric)
+      expiry = nil if expiry.is_a?(Numeric)
+      meta = {
+        key: key,
+        ttl: expiry&.to_i
+      }.compact.to_json
+
+      @internal.unset(key)
+      @internal.get_or_set(key, expiry: expiry) do
+        File.write(key_path(key), value)
+        File.write(meta_path(key), meta)
+        value
+      end
+    end
+
+    def delete(key)
+      @internal.unset(key)
+      @internal.lock do
+        [key_path(key), meta_path(key)].each do |path|
+          File.delete(path) if File.exist?(path)
+        end
+      end
+    end
+
+    def clear
+      @internal.clear
+      @internal.lock do
+        Dir[File.join(@dir, '**')].each_child { |entry| File.delete(File.join(@dir, entry)) }
+      end
+    end
+
+    private
+
+    def key_path(key)
+      File.join(@dir, safe_key(key))
+    end
+    def meta_path(key)
+      File.join(@dir, "#{safe_key(key)}.meta")
+    end
+
+    def key_valid?(key)
+      path = meta_path(key)
+      return unless File.exist?(path)
+
+      meta = JSON.parse(File.read(path), symbolize_names: true)
+      meta_valid?(meta)
+    end
+
+    def meta_valid?(meta)
+      return true unless meta[:ttl]
+      
+      Time.now < Time.at(meta[:ttl])
+    end
+
+    def safe_key(key)
+      key.gsub(%r{[^A-Za-z0-9-]}, '_')
+    end
+  end
+end

lib/fic_tracker/util/cache/memory.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+module FicTracker::Util::CacheImpl
+  # A threadsafe in-memory cache
+  class Memory < Base
+    Entry = Struct.new('Entry', :data, :ttl) do
+      def valid?
+        return true unless ttl
+
+        Time.now < ttl
+      end
+
+      def expired?
+        return false unless ttl
+
+        Time.now >= ttl
+      end
+    end
+
+    def initialize
+      @data = {}
+      @mutex = Thread::Mutex.new
+    end
+
+    def lock(&block)
+      @mutex.synchronize { block.call }
+    end
+
+    def expire
+      @mutex.synchronize do
+        @data.delete_if { |_, v| v.expired? }
+      end
+      nil
+    end
+
+    def has?(key)
+      @mutex.synchronize do
+        @data[key]&.valid?
+      end
+    end
+
+    def get(key)
+      @mutex.synchronize do
+        @data[key].data if has?(key)
+      end
+    end
+
+    def set(key, value, expiry = nil)
+      @mutex.synchronize do
+        expiry = expiry >= 0 ? Time.now + expiry : nil if expiry.is_a?(Numeric)
+        expiry = nil if expiry.is_a?(Numeric)
+
+        @data[key] = Entry.new(value, expiry)
+        value
+      end
+    end
+
+    def get_or_set(key, expiry: -1, &block)
+      @mutex.synchronize do
+        return @data[key].data if has?(key)
+
+        value = yield
+        expiry = expiry >= 0 ? Time.now + expiry : nil if expiry.is_a?(Numeric)
+        expiry = nil if expiry.is_a?(Numeric)
+
+        @data[key] = Entry.new(value, expiry)
+        value
+      end
+    end
+
+    def delete(key)
+      @mutex.synchronize do
+        @data.delete key
+      end
+      nil
+    end
+
+    def clear
+      @mutex.synchronize do
+        @data.clear
+      end
+      nil
+    end
+  end
+end

lib/fic_tracker/util/cache/redis.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require 'redis'
+
+module FicTracker::Util::CacheImpl
+  class Redis < Base
+    # A cache using live redis data
+    def initialize(namespace: 'cache', **redis)
+      @client = Redis.new(**redis)
+
+      @namespace = namespace
+    end
+
+    def has?(key)
+      @client.exists?(namespace_key(key))
+    end
+
+    def get(key)
+      @client.get(namespace_key(key))
+    end
+
+    def set(key, value, expiry = nil)
+      key = namespace_key(key)
+
+      expiry = expiry >= 0 ? Time.now + expiry : nil if expiry.is_a?(Numeric)
+      expiry = nil if expiry.is_a?(Numeric)
+
+      @client.set(key, value, exat: expiry)
+
+      value
+    end
+
+    def delete(key)
+      key = namespace_key(key)
+      @client.del key
+      nil
+    end
+
+    def clear
+      @client.scan_each(match: "#{@namespace}-*") { |key| @client.del key }
+    end
+
+    def each
+      @data.scan_each(match: "#{@namespace}-*") { |key| yield @client.get(key) }
+    end
+
+    private
+
+    def namespace_key(key)
+      "#{namespace}-#{key}"
+    end
+
+    def unnamespace_key(key)
+      key.delete_prefix("#{@namespace}-")
+    end
+  end
+end

lib/fic_tracker/util/database.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'sequel'
+require 'sequel/plugins/serialization'
+
+module FicTracker::Util
+  class Database
+    DEFAULT_SESSION_TIMEOUT = 48 * 60 * 60
+
+    def self.connect(connection_string, migrate: true, options: {})
+      raise 'Must provide a database URL' if connection_string.nil? || connection_string.empty?
+
+      db = if connection_string == :memory
+             Sequel.sqlite
+           else
+             Sequel.connect(connection_string, **options)
+           end
+
+      db.sql_log_level = :debug
+
+      # Defer logging until after migrations for in-memory store
+      db.loggers << Logging.logger[self] unless connection_string == :memory
+      self.migrate(db) if migrate
+      db.loggers << Logging.logger[self] if connection_string == :memory
+
+      db
+    end
+
+    def self.migrate(db)
+      Sequel.extension :migration
+      Sequel::Migrator.run(db, File.join(__dir__, '..', 'migrations'))
+    end
+  end
+end

lib/fic_tracker/util/hash_extensions.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+class Hash
+  def deep_transform_keys(&block)
+    to_h { |k, v| [block.call(k), v.is_a?(Hash) ? v.deep_transform_keys(&block) : v] }
+  end
+end

lib/fic_tracker/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module FicTracker
+  VERSION = "0.1.0"
+end

test/test_fic_tracker.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class TestFicTracker < Minitest::Test
+  def test_that_it_has_a_version_number
+    refute_nil ::FicTracker::VERSION
+  end
+
+  def test_it_does_something_useful
+    assert false
+  end
+end

test/test_helper.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+$LOAD_PATH.unshift File.expand_path("../lib", __dir__)
+require "fic_tracker"
+
+require "minitest/autorun"