From ff39289d6b2d0db31028133b91ee6d9dad23b5e2 Mon Sep 17 00:00:00 2001 From: Alexander Olofsson Date: Sat, 25 May 2024 23:31:26 +0200 Subject: [PATCH] Initial commit --- .github/workflows/main.yml | 27 +++ .gitignore | 12 ++ .rubocop.yml | 13 ++ CHANGELOG.md | 5 + Gemfile | 12 ++ LICENSE.txt | 21 ++ README.md | 16 ++ Rakefile | 12 ++ bin/fic_tracker | 73 +++++++ config.ru | 7 + config.yml.example | 6 + fic_tracker.gemspec | 35 +++ lib/fic_tracker.rb | 101 +++++++++ lib/fic_tracker/backend.rb | 96 +++++++++ lib/fic_tracker/backends/ao3/backend.rb | 204 ++++++++++++++++++ lib/fic_tracker/backends/ao3/client.rb | 100 +++++++++ .../backends/fimfiction/backend.rb | 44 ++++ lib/fic_tracker/backends/fimfiction/client.rb | 99 +++++++++ lib/fic_tracker/config.rb | 66 ++++++ lib/fic_tracker/converters/from_html.rb | 25 +++ lib/fic_tracker/converters/from_markdown.rb | 21 ++ lib/fic_tracker/converters/to_plain.rb | 32 +++ lib/fic_tracker/converters/to_safe_html.rb | 47 ++++ lib/fic_tracker/converters/to_simplemark.rb | 176 +++++++++++++++ lib/fic_tracker/migrations/001_create_base.rb | 180 ++++++++++++++++ lib/fic_tracker/models/author.rb | 48 +++++ lib/fic_tracker/models/chapter.rb | 98 +++++++++ lib/fic_tracker/models/light/search_info.rb | 30 +++ lib/fic_tracker/models/light/tag.rb | 58 +++++ lib/fic_tracker/models/story.rb | 187 ++++++++++++++++ lib/fic_tracker/models/user.rb | 12 ++ lib/fic_tracker/renderers/epub.rb | 200 +++++++++++++++++ lib/fic_tracker/renderers/html.rb | 140 ++++++++++++ lib/fic_tracker/renderers/markdown.rb | 69 ++++++ lib/fic_tracker/server.rb | 162 ++++++++++++++ lib/fic_tracker/tasks/cleanup.rb | 26 +++ lib/fic_tracker/util/cache.rb | 196 +++++++++++++++++ lib/fic_tracker/util/cache/base.rb | 11 + lib/fic_tracker/util/cache/database.rb | 60 ++++++ lib/fic_tracker/util/cache/dummy.rb | 7 + lib/fic_tracker/util/cache/file.rb | 99 +++++++++ lib/fic_tracker/util/cache/memory.rb | 85 ++++++++ lib/fic_tracker/util/cache/redis.rb | 57 +++++ lib/fic_tracker/util/database.rb | 35 +++ lib/fic_tracker/util/hash_extensions.rb | 7 + lib/fic_tracker/version.rb | 5 + test/test_fic_tracker.rb | 13 ++ test/test_helper.rb | 6 + views/author.haml | 0 views/chapter.haml | 0 views/index.haml | 0 views/story.haml | 0 52 files changed, 3041 insertions(+) create mode 100644 .github/workflows/main.yml create mode 100644 .gitignore create mode 100644 .rubocop.yml create mode 100644 CHANGELOG.md create mode 100644 Gemfile create mode 100644 LICENSE.txt create mode 100644 README.md create mode 100644 Rakefile create mode 100755 bin/fic_tracker create mode 100644 config.ru create mode 100644 config.yml.example create mode 100644 fic_tracker.gemspec create mode 100644 lib/fic_tracker.rb create mode 100644 lib/fic_tracker/backend.rb create mode 100644 lib/fic_tracker/backends/ao3/backend.rb create mode 100644 lib/fic_tracker/backends/ao3/client.rb create mode 100644 lib/fic_tracker/backends/fimfiction/backend.rb create mode 100644 lib/fic_tracker/backends/fimfiction/client.rb create mode 100644 lib/fic_tracker/config.rb create mode 100644 lib/fic_tracker/converters/from_html.rb create mode 100644 lib/fic_tracker/converters/from_markdown.rb create mode 100644 lib/fic_tracker/converters/to_plain.rb create mode 100644 lib/fic_tracker/converters/to_safe_html.rb create mode 100644 lib/fic_tracker/converters/to_simplemark.rb create mode 100644 lib/fic_tracker/migrations/001_create_base.rb create mode 100644 lib/fic_tracker/models/author.rb create mode 100644 lib/fic_tracker/models/chapter.rb create mode 100644 lib/fic_tracker/models/light/search_info.rb create mode 100644 lib/fic_tracker/models/light/tag.rb create mode 100644 lib/fic_tracker/models/story.rb create mode 100644 lib/fic_tracker/models/user.rb create mode 100644 lib/fic_tracker/renderers/epub.rb create mode 100644 lib/fic_tracker/renderers/html.rb create mode 100644 lib/fic_tracker/renderers/markdown.rb create mode 100644 lib/fic_tracker/server.rb create mode 100644 lib/fic_tracker/tasks/cleanup.rb create mode 100644 lib/fic_tracker/util/cache.rb create mode 100644 lib/fic_tracker/util/cache/base.rb create mode 100644 lib/fic_tracker/util/cache/database.rb create mode 100644 lib/fic_tracker/util/cache/dummy.rb create mode 100644 lib/fic_tracker/util/cache/file.rb create mode 100644 lib/fic_tracker/util/cache/memory.rb create mode 100644 lib/fic_tracker/util/cache/redis.rb create mode 100644 lib/fic_tracker/util/database.rb create mode 100644 lib/fic_tracker/util/hash_extensions.rb create mode 100644 lib/fic_tracker/version.rb create mode 100644 test/test_fic_tracker.rb create mode 100644 test/test_helper.rb create mode 100644 views/author.haml create mode 100644 views/chapter.haml create mode 100644 views/index.haml create mode 100644 views/story.haml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..7fb6379 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,27 @@ +name: Ruby + +on: + push: + branches: + - master + + pull_request: + +jobs: + build: + runs-on: ubuntu-latest + name: Ruby ${{ matrix.ruby }} + strategy: + matrix: + ruby: + - '3.2.3' + + steps: + - uses: actions/checkout@v4 + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby }} + bundler-cache: true + - name: Run the default task + run: bundle exec rake diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ebe157b --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +/.bundle/ +/.yardoc +/_yardoc/ +/coverage/ +/doc/ +/pkg/ +/spec/reports/ +/tmp/ +/vendor/ +/Gemfile.lock +/database.db +/config.yml diff --git a/.rubocop.yml b/.rubocop.yml new file mode 100644 index 0000000..1a44467 --- /dev/null +++ b/.rubocop.yml @@ -0,0 +1,13 @@ +AllCops: + TargetRubyVersion: 3.2 + +Style/StringLiterals: + Enabled: true + # EnforcedStyle: single_quotes + +Style/StringLiteralsInInterpolation: + Enabled: true + # EnforcedStyle: single_quotes + +Layout/LineLength: + Max: 120 diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..b68d1e6 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,5 @@ +## [Unreleased] + +## [0.1.0] - 2024-04-25 + +- Initial release diff --git a/Gemfile b/Gemfile new file mode 100644 index 0000000..3f8b1cc --- /dev/null +++ b/Gemfile @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +# Specify your gem's dependencies in fic_tracker.gemspec +gemspec + +gem "rake", "~> 13.0" + +gem "minitest", "~> 5.16" + +gem "rubocop", "~> 1.21" diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..6c9975e --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2024 Alexander Olofsson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..c3cc100 --- /dev/null +++ b/README.md @@ -0,0 +1,16 @@ +# FicTracker + +A web application to handle tracking and reading live fiction, as it's updated. + + +## Usage + +Instantiate config.yml and use rackup + +## Contributing + +Bug reports and pull requests are welcome on GitHub at https://github.com/ananace/ruby-fic_tracker. + +## License + +The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT). diff --git a/Rakefile b/Rakefile new file mode 100644 index 0000000..2bf771f --- /dev/null +++ b/Rakefile @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +require "bundler/gem_tasks" +require "minitest/test_task" + +Minitest::TestTask.create + +require "rubocop/rake_task" + +RuboCop::RakeTask.new + +task default: %i[test rubocop] diff --git a/bin/fic_tracker b/bin/fic_tracker new file mode 100755 index 0000000..4bf9493 --- /dev/null +++ b/bin/fic_tracker @@ -0,0 +1,73 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require 'fic_tracker' +require 'optparse' +require 'ostruct' + +options = OpenStruct.new +OptParse.new do |opts| + opts.banner = 'Usage: fic_tracker [OPTIONS...]' + + opts.on '-b', '--backend=BACKEND', 'The backend to use' do |backend| + options.backend = backend + end + + opts.on '-s', '--story=STORY', 'The story to download' do |story| + options.story = story + end + + opts.on '-c', '--chapter=CHAPTER', 'The chapter to download' do |chapter| + options.chapter = chapter + end + + opts.on '-f', '--format=FORMAT', 'The format to download (epub, html, markdown)' do |format| + options.format = format.to_sym + end + + opts.on '-o', '--output=FILE', 'The resulting file to save the download into' do |output| + options.output = output + end + + opts.separator '' + + opts.on '-C', '--config=FILE', 'Specify a configuration file to read' do |config| + options.config = config + end + + opts.on '-h', '--help', 'Show this text' do + puts opts + exit + end + + opts.on '-q', '--quiet', 'Run quietly' do + options.log_level = :error + end + + opts.on '-v', '--verbose', 'Run verbosely, can be specified twice for debug output' do + if %i[info debug].include?(options.log_level) + options.log_level = :debug + else + options.log_level = :info + end + end + + opts.on '-V', '--version', 'Prints the version and exits' do + puts FicTracker::VERSION + exit + end +end.parse! + +FicTracker.logger.level = options.log_level || :warn +FicTracker.configure! + +backend = FicTracker::Backends.get(options.backend) +story = FicTracker::Models::Story.find(backend_name: backend.name, slug: backend.parse_slug(options.story)) || FicTracker::Models::Story.new(backend: backend, slug: backend.parse_slug(options.story)) +story.ensure_fully_loaded +data = nil + +options.output ||= "#{story.safe_name}.#{options.format}" +FicTracker.logger.info "Saving to #{options.output}" +File.open(options.output, 'w') { |f| FicTracker::Renderers.render(options.format, story, io: f) } + +story.save_changes diff --git a/config.ru b/config.ru new file mode 100644 index 0000000..732d2ac --- /dev/null +++ b/config.ru @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'fic_tracker' + +FicTracker.configure! + +map '/' ->() { run FicTracker::Server } diff --git a/config.yml.example b/config.yml.example new file mode 100644 index 0000000..97328e3 --- /dev/null +++ b/config.yml.example @@ -0,0 +1,6 @@ +--- +database: + url: sqlite://database.db + +cache: + type: database diff --git a/fic_tracker.gemspec b/fic_tracker.gemspec new file mode 100644 index 0000000..9b5b60a --- /dev/null +++ b/fic_tracker.gemspec @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +require_relative "lib/fic_tracker/version" + +Gem::Specification.new do |spec| + spec.name = "fic_tracker" + spec.version = FicTracker::VERSION + spec.authors = ["Alexander Olofsson"] + spec.email = ["ace@haxalot.com"] + + spec.summary = 'A tracker and aggregator for live fiction' + spec.description = 'A web service that tracks and aggregates live fiction for use with e-readers or similar devices' + spec.homepage = 'https://github.com/ananace/ruby-fic_tracker' + spec.license = 'MIT' + spec.required_ruby_version = '>= 3.0.0' + + spec.metadata["allowed_push_host"] = '' + + spec.metadata["homepage_uri"] = spec.homepage + spec.metadata["source_code_uri"] = spec.homepage + spec.metadata["changelog_uri"] = "#{spec.homepage}/blob/master/CHANGELOG.md" + + spec.files = Dir['{lib,views,public}/**/*'] + %w[CHANGELOG.md LICENSE.txt README.md config.ru config.yml.example fic_tracker.gemspec] + + spec.add_dependency 'haml' + spec.add_dependency 'kramdown' + spec.add_dependency 'logging' + spec.add_dependency 'nokogiri' + spec.add_dependency 'rackup' + spec.add_dependency 'ruby-vips' + spec.add_dependency 'rubyzip' + spec.add_dependency 'sequel' + spec.add_dependency 'sinatra' + spec.add_dependency 'sqlite3' +end diff --git a/lib/fic_tracker.rb b/lib/fic_tracker.rb new file mode 100644 index 0000000..4a61063 --- /dev/null +++ b/lib/fic_tracker.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +require 'logging' + +require_relative 'fic_tracker/version' + +module FicTracker + class Error < StandardError; end + + class ConfigError < Error; end + + def self.logger + @logger ||= Logging.logger[self].tap do |logger| + logger.add_appenders ::Logging.appenders.stdout + logger.level = :warn + end + end + + def self.configure! + Config.load! + end + + def self.cache + configure! unless @cache + @cache + end + + def self.cache=(cache) + @cache = cache + end + + def self.database + configure! unless @database + @database + end + + def self.database=(database) + @database = database + end + + def self.debug! + logger.level = :debug + end + + def self.logger=(global_logger) + @logger = global_logger + @global_logger = !global_logger.nil? + end + + def self.global_logger? + @global_logger ||= false + end + + module Models + def self.const_missing(const) + raise 'No database connected' unless FicTracker.database + + model = const.to_s.downcase + require_relative "fic_tracker/models/#{model}" + + mod = const_get(const) if const_defined? const + return mod if mod + + raise "Model not found: #{const}" + end + end + + module Tasks + autoload :Cleanup, 'fic_tracker/tasks/cleanup' + end + + module Renderers + autoload :Epub, 'fic_tracker/renderers/epub' + autoload :HTML, 'fic_tracker/renderers/html' + autoload :Markdown, 'fic_tracker/renderers/markdown' + + def self.render(type, story, **attrs) + klass = case type + when :Markdown, :markdown, :md + Markdown + when :HTML, :html + HTML + when :Epub, :epub + Epub + end + + unless attrs[:io] + require 'stringio' + attrs[:io] = StringIO.new + attrs[:_stringio] = true + end + + return klass.new(story, **attrs).render unless attrs[:stringio] + + attrs[:io].string + end + end +end + +require_relative 'fic_tracker/backend' +require_relative 'fic_tracker/config' diff --git a/lib/fic_tracker/backend.rb b/lib/fic_tracker/backend.rb new file mode 100644 index 0000000..c0e029e --- /dev/null +++ b/lib/fic_tracker/backend.rb @@ -0,0 +1,96 @@ +# frozen_string_literal: true + +module FicTracker + class Backend + class SearchInfo + attr_accessor :tag_categories + end + + def self.backend_name + name.split('::')[-2] + end + + def self.config_name + backend_name.downcase.to_sym + end + + def name + self.class.backend_name + end + + def full_name + self.class.name + end + + def url; end + + def load_author(author); end + def load_story(slug); end + def find_chapters(story); end + def load_full_story(story) + story = load_story(story) + find_chapters(story) + story.chapters.each { |c| c.refresh_content } + story + end + def load_chapter(slug, story); end + + def get_search_info; end + def search(search_info); end + + def parse_slug(slug) + slug + end + + def cache_key + [ self.class.name.split('::').map(&:downcase)[-2, 1] ] + end + + protected + + def logger + Logging.logger[self] + end + + def cache + FicTracker.cache + end + end + + module Backends + def self.get(name) + const = name + const = const.to_s.to_sym unless const.is_a? Symbol + + model = constants.find { |c| c == const } + return const_get(model).const_get(:Backend).new if model + + model = load(name) + return model if model + + raise "Unknown backend #{name}" + end + + def self.const_missing(const) + mod = load(const) + return mod if mod + + raise "Unknown backend #{const}" + end + + def self.load(name) + model = name.to_s.downcase + require_relative "backends/#{model}/backend" + + model = constants.find { |c| c.to_s.downcase == model } + return unless model + + backend = const_get(model).const_get(:Backend) + backend.new(**FicTracker.config.dig(:backends, backend.config_name, default: {})) + rescue StandardError => e + Logging.logger[backend].error "Failed to load, #{e.class}: #{e}" + + nil + end + end +end diff --git a/lib/fic_tracker/backends/ao3/backend.rb b/lib/fic_tracker/backends/ao3/backend.rb new file mode 100644 index 0000000..7a8c389 --- /dev/null +++ b/lib/fic_tracker/backends/ao3/backend.rb @@ -0,0 +1,204 @@ +# frozen_string_literal: true + +require_relative 'client' + +module FicTracker::Backends::Ao3 + class Backend < FicTracker::Backend + TAG_ORDERING = { + rating: -3, + category: -2, + warning: -1, + }.freeze + + def client + @client ||= Client.new + end + + def full_name + 'Archive of Our Own' + end + + def url + Client::BASE_URL + end + + def load_author(author) + author = FicTracker::Models::Author.new(slug: parse_slug(author), backend: self) unless author.is_a? FicTracker::Models::Author + + logger.info "Loading author #{author.slug}" + doc = client.request("/users/#{author.slug}") + + user = doc.at_css('#main .user') + + url = user.at_css('.icon a') + url = URI.join(Client::BASE_URL, url[:href]) if url + image = user.at_css('.icon a img') + image = URI.join(Client::BASE_URL, image[:src]) if image + + author.set( + name: user.at_css('h2.heading').text.strip, + url: url.to_s, + image: image.to_s, + last_metadata_refresh: Time.now + ) + end + + def load_story(story) + story = FicTracker::Models::Story.new(slug: parse_slug(story), backend: self) unless story.is_a? FicTracker::Models::Story + + logger.info "Loading story #{story.slug}" + doc = client.request("/works/#{story.slug}") + + attrs = extract_story doc + + story.set(last_metadata_refresh: Time.now, **attrs) + end + + def find_chapters(story) + story = FicTracker::Models::Story.new(slug: parse_slug(story), backend: self) unless story.is_a? FicTracker::Models::Story + + logger.info "Loading chapters for #{story.slug}" + doc = client.request("/works/#{story.slug}/navigate") + + chapters = doc.at_css('ol.chapter').css('li').map do |entry| + published_at = Time.parse(entry.at_css('span.datetime').text.strip) + link = entry.at_css('a') + index, *name = link.text.split('. ') + index = index.to_i + name = name.join('. ') + url = URI.join(Client::BASE_URL, link[:href]) + slug = url.path.split('/').last + + { + slug: slug, + index: index, + name: name, + url: url.to_s, + published_at: published_at, + } + end + + story.set( + last_content_refresh: Time.now, + ) + story.chapters = chapters + + story + end + + def load_full_story(story) + story = FicTracker::Models::Story.new(slug: parse_slug(story), backend: self) unless story.is_a? FicTracker::Models::Story + + logger.info "Loading all chapters for #{story.slug}" + doc = client.request("/works/#{story.slug}", query: { view_full_work: true }) + + attrs = extract_story(doc) + chapters = doc.css('#chapters > div.chapter').map { |chapter| extract_chapter(chapter) } + + story.set( + last_metadata_refresh: Time.now, + last_content_refresh: Time.now, + **attrs + ) + story.chapters = chapters + + story + end + + def load_chapter(chapter, story) + story = load_story(story) unless story.is_a? FicTracker::Models::Story + chapter = FicTracker::Models::Chapter.new(slug: parse_slug(chapter), story: story) unless chapter.is_a? FicTracker::Models::Chapter + + logger.info "Loading chapter #{chapter.slug} for #{story.slug}" + doc = client.request("/works/#{story.slug}/chapters/#{chapter.slug}") + + attrs = extract_chapter(doc.at_css('#chapters > div.chapter')) + + chapter.set(**attrs) + end + + def parse_slug(slug) + return URI(slug).path.split('/').last if slug.is_a?(String) && slug.start_with?('http') + + slug.to_s + end + + private + + def extract_story(doc) + url = URI.join(Client::BASE_URL, doc.at_css('li.share a')[:href].delete_suffix('/share')) + meta = doc.at_css('#main dl.meta') + preface = doc.at_css('#workskin .preface') + + name = preface.at_css('.title').text.strip + synopsis = preface.at_css('.summary blockquote').children.to_xml + language = meta.at_css('dd.language')['lang'] + author = preface.at_css('a[rel="author"]')[:href].split('/')[2] + + tags = meta.css('dd.tags').map do |tagblock| + category = tagblock[:class].split.first.to_sym + + tagblock.css('a.tag').map do |a| + next if ['Creator Chose Not To Use Archive Warnings', 'No Archive Warnings Apply'].include? a.text.strip + + { + name: a.text.strip, + category: category, + important: %i[rating warning category].include?(category) ? true : nil, + ordering: TAG_ORDERING[category], + }.compact + end.compact + end.flatten + + chapters = meta.at_css('dl.stats dd.chapters').text.strip.split('/') + words = meta.at_css('dl.stats dd.words').text.strip.tr(',', '').to_i + published_at = meta.at_css('dl.stats dd.published')&.text&.strip + published_at = Time.parse(published_at) if published_at + updated_at = meta.at_css('dl.stats dd.status')&.text&.strip + updated_at = Time.parse(updated_at) if updated_at + + { + name: name, + author: author, + synopsis: synopsis, + url: url.to_s, + language: language, + chapter_count: chapters.first.to_i, + word_count: words, + completed: chapters.first == chapters.last, + published_at: published_at, + updated_at: updated_at, + tags: FicTracker::Models::Light::Tag.load(tags), + } + end + + def extract_chapter(chapter_doc) + link = chapter_doc.at_css('h3.title a') + url = URI.join(Client::BASE_URL, link[:href]) + slug = url.path.split('/').last + index = link.text.strip.split.last.to_i + + html = chapter_doc.at_css('div[role="article"]').children[2..].map(&:to_xml).join("\n") + + title = chapter_doc.at_css('h3.title') + if title + title_base = title.at_css('a').text.strip + title_extra = title.text.strip.delete_prefix("#{title_base}:").strip + + title = title_extra.empty? ? title_base : title_extra + end + + { + index: index, + slug: slug, + + name: title, + url: url.to_s, + last_refresh: Time.now, + + content: html, + content_type: 'text/html', + } + end + end +end diff --git a/lib/fic_tracker/backends/ao3/client.rb b/lib/fic_tracker/backends/ao3/client.rb new file mode 100644 index 0000000..ab3928e --- /dev/null +++ b/lib/fic_tracker/backends/ao3/client.rb @@ -0,0 +1,100 @@ +# frozen_string_literal: true + +require 'cgi' +require 'net/http' +require 'nokogiri' +require 'json' + +module FicTracker::Backends::Ao3 + class Client + BASE_URL = 'https://archiveofourown.org' + + def url + URI(BASE_URL) + end + + def request(path, type: :html, body: nil, query: nil, method: :get, redirect: true) + uri = URI.join(url, path) + uri.query = URI.encode_www_form(query) if query + + req = Net::HTTP.const_get(method.to_s.capitalize.to_sym).new uri.request_uri + req['User-Agent'] = "FicTracker/#{FicTracker::VERSION}" + case type + when :html, :xml + req['Accept'] = 'text/html,application/xhtml+xml,application/xml' + when :json + req['Accept'] = 'application/json' + end + + if body + req.body = body + req.body = req.body.to_json unless req.body.is_a? String + req['Content-Type'] = 'application/json' + end + + resp = nil + http.start do + loop do + debug_http(req) + resp = http.request req + debug_http(resp) + case resp + when Net::HTTPRedirection + req.path.replace resp['location'] + when Net::HTTPTooManyRequests + wait_time = 10 + if resp['retry-after'] + after = resp['retry-after'] + wait_time = after =~ /\A[0-9]+\Z/ ? after.to_i : Time.parse(after) + end + wait_time = Time.now + wait_time if wait_time.is_a? Numeric + + logger.info "Rate limited, waiting until #{wait_time} before retrying" + sleep Time.now - wait_time + else + break + end + end + end + + resp.value + return if resp.body.empty? + + case type + when :html + Nokogiri::HTML4.parse(resp.body) + when :xml + Nokogiri::XML.parse(resp.body) + when :json + JSON.parse(resp.body, symbolize_names: true) + when :raw + resp.body + end + end + + private + + def logger + Logging.logger[self] + end + + def http + @http ||= Net::HTTP.new(url.host, url.port).tap { |http| http.use_ssl = url.scheme == 'https' } + end + + def debug_http(object) + return unless logger.debug? + + dir = '>' + if object.is_a?(Net::HTTPRequest) + dir = '<' + + logger.debug "#{dir} #{object.method} #{object.path}" + else + logger.debug "#{dir} #{object.code} #{object.message}" + end + + object.each_header { |h, v| logger.debug "#{dir} #{h}: #{v}" } + end + end +end diff --git a/lib/fic_tracker/backends/fimfiction/backend.rb b/lib/fic_tracker/backends/fimfiction/backend.rb new file mode 100644 index 0000000..6f8fdbe --- /dev/null +++ b/lib/fic_tracker/backends/fimfiction/backend.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +require_relative 'fimfiction/client' + +module FicTracker::Backends::FIMFiction + class Backend < FicTracker::Backend + def client + @client ||= Client.new + end + + def full_name + 'FIM Fiction' + end + + def url + 'https://www.fimfiction.net' + end + + def load_author(author) + author = FicTracker::Models::Author.new(slug: author, backend: self) unless author.is_a? FicTracker::Models::Author + + logger.info "Loading author #{author.slug}" + end + def load_story(slug) + story = FicTracker::Models::Story.new(slug: story, backend: self) unless story.is_a? FicTracker::Models::Story + + logger.info "Loading story #{story.slug}" + end + def find_chapters(story) + story = FicTracker::Models::Story.new(slug: story, backend: self) unless story.is_a? FicTracker::Models::Story + + logger.info "Loading chapters for #{story.slug}" + end + def load_chapter(chapter, story) + story = FicTracker::Models::Story.new(slug: story, backend: self) unless story.is_a? FicTracker::Models::Story + chapter = FicTracker::Models::Chapter.new(slug: chapter, story: story) unless chapter.is_a? FicTracker::Models::Chapter + + logger.info "Loading chapter #{chapter.slug} for #{story.slug}" + end + + def get_search_info; end + def search(search_info); end + end +end diff --git a/lib/fic_tracker/backends/fimfiction/client.rb b/lib/fic_tracker/backends/fimfiction/client.rb new file mode 100644 index 0000000..2d2fa50 --- /dev/null +++ b/lib/fic_tracker/backends/fimfiction/client.rb @@ -0,0 +1,99 @@ +# frozen_string_literal: true + +require 'cgi' +require 'json' +require 'net/http' +require 'uri' + +module FicTracker::Backends::FIMFiction + class Client + BASE_URL = URI('https://www.fimfiction.net/api/v2').freeze + + attr_accessor :access_token, :url + attr_reader :tags + + def initialize + @access_token = nil + @url = BASE_URL + + @tags = Tags.new(self) + end + + def request(path, body: nil, query: nil, method: :get) + uri = URI.join(BASE_URL, path) + uri.query = URI.encode_www_form(query) if query + + req = Net::HTTP.const_get(method.to_s.capitalize.to_sym).new + req['Accept'] = 'application/vnd.api+json' + req['Authorization'] = "Bearer #{access_token}" if access_token + req['User-Agent'] = "FicTracker/#{FicTracker::VERSION}" + + if body + req.body = body + req.body = req.body.to_json unless req.body.is_a? String + req['Content-Type'] = 'application/json' + end + + resp = nil + http.start do + loop do + debug_http(req) + resp = http.request req + debug_http(resp) + case resp + when Net::HTTPRedirection + req.path.replace resp['location'] + when Net::HTTPTooManyRequests + wait_time = 10 + if resp['retry-after'] + after = resp['retry-after'] + wait_time = after =~ /\A[0-9]+\Z/ ? after.to_i : Time.parse(after) + end + wait_time = Time.now + wait_time if wait_time.is_a? Numeric + + logger.info "Rate limited, waiting until #{wait_time} before retrying" + sleep Time.now - wait_time + else + break + end + end + end + + resp.value + return if resp.body.empty? + + return JSON.parse(resp.body, symbolize_names: true) + end + + def retrieve_token(id, secret) + data = request('/token', body: { client_id: id, client_secret: secret, grant_type: :client_credentials }, method: :post) + + @access_token = data[:access_token] + end + + private + + def logger + Logging.logger[self] + end + + def http + @http ||= Net::HTTP.new(url.host, url.port).tap { |http| http.use_ssl = url.scheme == 'https' } + end + + def debug_http(object) + return unless logger.debug? + + dir = '>' + if object.is_a?(Net::HTTPRequest) + dir = '<' + + logger.debug "#{dir} #{object.method} #{object.path}" + else + logger.debug "#{dir} #{object.code} #{object.message}" + end + + object.each_header { |h, v| logger.debug "#{dir} #{h}: #{v}" } + end + end +end diff --git a/lib/fic_tracker/config.rb b/lib/fic_tracker/config.rb new file mode 100644 index 0000000..d5de1bb --- /dev/null +++ b/lib/fic_tracker/config.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +require 'psych' + +require_relative 'util/cache' +require_relative 'util/database' + +module FicTracker::Config + class << self + def load!(config_file: 'config.yml') + @config_file = config_file + @config = {} + + load_internal + + FicTracker.database = FicTracker::Util::Database.connect( + dig(:database, :url, default: :memory), + migrate: dig(:database, :migrate, default: true), + options: dig(:database, :options, default: {}) + ) + FicTracker.cache = FicTracker::Util::Cache.create( + type: dig(:cache, :type, default: :none), + compress: dig(:cache, :compress, default: nil), + encoding: dig(:cache, :encoding, default: nil), + options: dig(:cache, :options, default: {}) + ) + + Sequel::Model.db = FicTracker.database + end + + def [](arg) + @config[arg] + end + + def dig(*args, default: nil) + envvar = (%i[ft] + args.map { |arg| arg.to_s }).join('_').upcase + return ENV[envvar] if ENV[envvar] + + ret = @config.dig(*args) + return ret if ret + return default unless default.nil? + return yield if block_given? + + nil + end + + private + + def load_internal + @config_file = ENV['FT_CONFIG_FILE'] if ENV['FT_CONFIG_FILE'] + begin + @config = Psych.load(File.read(@config_file)).deep_transform_keys(&:to_sym) + rescue + @config = {} + end + @config[:database] ||= {} + @config[:database][:config] ||= {} + @config[:cache] ||= {} + + if ENV['FT_CACHE_REDIS_URL'] + @config[:cache][:type] = :redis + @config[:cache][:url] = ENV['FT_CACHE_REDIS_URL'] + end + end + end +end diff --git a/lib/fic_tracker/converters/from_html.rb b/lib/fic_tracker/converters/from_html.rb new file mode 100644 index 0000000..759589d --- /dev/null +++ b/lib/fic_tracker/converters/from_html.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +require 'kramdown' + +module FicTracker::Converters + class FromHTML + def self.to_plain(content) + require_relative 'to_plain' + + Kramdown::Document.new(content, input: 'html').to_plain.strip + end + + def self.to_md(content) + require_relative 'to_simplemark' + + Kramdown::Document.new(content, input: 'html').to_simplemark.strip + end + + def self.to_safe_html(content) + require_relative 'to_safe_html' + + Kramdown::Document.new(content, input: 'html').to_safe_html + end + end +end diff --git a/lib/fic_tracker/converters/from_markdown.rb b/lib/fic_tracker/converters/from_markdown.rb new file mode 100644 index 0000000..90607a6 --- /dev/null +++ b/lib/fic_tracker/converters/from_markdown.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +require 'kramdown' + +module FicAggregator::Converters + class FromMarkdown + def self.to_plain(content) + require_relative 'to_plain' + + Kramdown::Document.new(content, input: 'markdown').to_plain.strip + end + + def self.to_html(content) + Kramdown::Document.new(content, input: 'markdown').to_html.strip + end + + def self.to_safe_html(content) + Kramdown::Document.new(content, input: 'markdown').to_safe_html + end + end +end diff --git a/lib/fic_tracker/converters/to_plain.rb b/lib/fic_tracker/converters/to_plain.rb new file mode 100644 index 0000000..858dd24 --- /dev/null +++ b/lib/fic_tracker/converters/to_plain.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +require_relative 'to_simplemark' + +module Kramdown::Converter + # Markdown with some slight simplification to ease reading + class Plain < Simplemark + private + + def convert_header(el, depth = 1) + depth = 7 - depth + "#{'=' * depth} #{inner(el)} #{'=' * depth}\n\n" + end + + def convert_strong(el) + "*#{inner(el)}*" + end + + def convert_em(el) + "_#{inner(el)}_" + end + + def convert_a(el) + content = inner(el) + [content, "(#{el.attr['href']})"].reject { |k| k.nil? || k.empty? }.join ' ' + end + + def convert_blockquote(el) + " #{inner(el).split("\n").join("\n ")}" + end + end +end diff --git a/lib/fic_tracker/converters/to_safe_html.rb b/lib/fic_tracker/converters/to_safe_html.rb new file mode 100644 index 0000000..2beba9c --- /dev/null +++ b/lib/fic_tracker/converters/to_safe_html.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +require 'kramdown' +require 'logging' + +module Kramdown::Converter + # HTML converter that strips out elements not relevant for e-readers + class SafeHtml < Base + def initialize(root, options) + super + + @options[:remove_block_html_tags] = true + @options[:remove_span_html_tags] = false + @options[:template] = 'string://<%= Html.convert(@body).first %>' + end + + SUPERFLUOUS_TAGS = %w[align class justify] + + def convert(el) + real_el, el = el, el.value if el.type == :footnote + + # Strip out unnecessary HTML tags + SUPERFLUOUS_TAGS.each { |tag| el.attr.delete tag if el.attr.key? tag } + + children = el.children.dup + index = 0 + + while index < children.length + if %i[img xml_pi].include?(children[index].type) || + (children[index].type == :html_element && %w[style script title].include?(children[index].value)) + children[index..index] = [] + elsif children[index].type == :html_element && ( + (@options[:remove_block_html_tags] && children[index].options[:category] == :block) || + (@options[:remove_span_html_tags] && children[index].options[:category] == :span) + ) && children[index].value != 'div' + children[index..index] = children[index].children + else + convert(children[index]) + index += 1 + end + end + el.children = children + + real_el || el + end + end +end diff --git a/lib/fic_tracker/converters/to_simplemark.rb b/lib/fic_tracker/converters/to_simplemark.rb new file mode 100644 index 0000000..cb6a4de --- /dev/null +++ b/lib/fic_tracker/converters/to_simplemark.rb @@ -0,0 +1,176 @@ +# frozen_string_literal: true + +require 'kramdown' +require 'logging' + +module Kramdown::Converter + # Simplified commonmark converter + class Simplemark < Base + def convert(el, **_) + meth = :"convert_#{el.type}" + meth = :"convert_html_#{el.value}" if el.type == :html_element + + if self.class.private_method_defined? meth + res = send(meth, el) + else + fallback = :convert_blank + fallback = :convert_html_element if el.type == :html_element + + logger.debug "Missing #{meth.inspect} function, using #{fallback}" + res = send(fallback, el) + end + res = res.dup if res.frozen? + res + end + + private + + def logger + Logging.logger[self] + end + + class UlBullet + def next + '-' + end + end + + class OlBullet + def initialize + @at = 1 + end + + def next + "#{@at}." + ensure + @at = @at.succ! + end + end + + def inner(el) + el.children.map { |el| convert(el) }.join + end + alias convert_root inner + + def convert_blank(el) + '' + end + alias convert_comment convert_blank + + def convert_p(el) + "#{inner(el)}\n\n" + end + + def convert_text(el) + el.value + end + alias convert_abbreviation convert_text + + def convert_strong(el) + "__#{inner(el).gsub('__', '\_\_')}__" + end + alias convert_html_b convert_strong + alias convert_html_strong convert_strong + + def convert_em(el) + "_#{inner(el).gsub('_', '\_')}_" + end + alias convert_html_i convert_em + alias convert_html_em convert_em + + def convert_del(el) + "~#{inner(el)}~" + end + alias convert_html_del convert_del + + def convert_codespan(el) + "`#{el.value}`" + end + alias convert_html_code convert_codespan + + def convert_blockquote(el) + "> #{inner(el).split("\n").join("\n> ")}\n\n" + end + + def convert_codeblock(el) + "```\n#{el.value}```\n\n" + end + + def convert_a(el) + content = inner(el) + "[#{content}](#{el.attr['href']})" + end + alias convert_html_a convert_a + + def convert_header(el, depth = 1) + "#{'#' * depth} #{inner(el)} #{'#' * depth}\n\n" + end + def convert_html_h1(el); convert_header(el, 1) end + def convert_html_h2(el); convert_header(el, 2) end + def convert_html_h3(el); convert_header(el, 3) end + def convert_html_h4(el); convert_header(el, 4) end + def convert_html_h5(el); convert_header(el, 5) end + def convert_html_h6(el); convert_header(el, 6) end + + def convert_br(el) + " \n" + end + + def convert_hr(el) + "#{'-' * 16}\n\n" + end + + def convert_ul(el) + @bullet = UlBullet.new + "#{inner(el)}\n" + ensure + @bullet = nil + end + alias convert_html_ul convert_ul + + def convert_ol(el) + @bullet = OlBullet.new + "#{inner(el)}\n" + ensure + @bullet = nil + end + alias convert_html_ol convert_ol + + def convert_li(el) + " #{@bullet.next} #{inner(el).strip}\n" + end + alias convert_html_li convert_li + + def convert_smart_quote(el) + SMART_QUOTES.fetch(el.value) + end + + def convert_typographic_sym(el) + TYPOGRAPHIC_SYMBOLS.fetch(el.value) + end + + def convert_entity(el) + el.value.char + end + + alias convert_html_title convert_blank + + # Strip out unknown HTML elements, only keep content + alias convert_html_element inner + + SMART_QUOTES = { + lsquo: "‘", + rsquo: "’", + ldquo: "“", + rdquo: "”" }.freeze + + TYPOGRAPHIC_SYMBOLS = { + mdash: "—", + ndash: "–", + hellip: "...", + laquo_space: "« ", + raquo_space: " »", + laquo: "«", + raquo: "»" }.freeze + end +end diff --git a/lib/fic_tracker/migrations/001_create_base.rb b/lib/fic_tracker/migrations/001_create_base.rb new file mode 100644 index 0000000..705b465 --- /dev/null +++ b/lib/fic_tracker/migrations/001_create_base.rb @@ -0,0 +1,180 @@ +# frozen_string_literal: true + +Sequel.migration do + change do + create_table(:meta) do + String :key, null: false, primary_key: true + String :value, null: true + end + + create_table(:cache) do + String :key, null: false, primary_key: true + File :value, null: true + DateTime :expire_at, null: true, default: nil + end + + create_table(:authors) do + primary_key :id + String :slug, null: false + String :backend_name, null: false + index %i[slug backend_name], unique: true + + String :name, null: false + + String :url, null: true, default: nil + String :image, null: true, default: nil + + DateTime :last_metadata_refresh, null: true, default: nil + + # JSON + String :data, null: false, text: true, default: '{}' + end + + create_table(:chapters) do + primary_key :id + + foreign_key :story_id, :stories, on_delete: :cascade, on_update: :cascade + Integer :index, null: false + String :slug, null: false + index %i[story_id index], unique: true + index %i[story_id slug], unique: true + + String :etag, null: true, default: nil + String :name, null: true, default: nil + + String :url, null: true, default: nil + + DateTime :published_at, null: true, default: nil + DateTime :updated_at, null: true, default: nil + DateTime :last_refresh, null: true, default: nil + + # JSON + String :data, null: false, text: true, default: '{}' + end + + create_table(:collections) do + primary_key :id + String :slug, null: false + String :backend_name, null: false + index %i[slug backend_name], unique: true + + String :type, null: false + String :name, null: true, default: nil + String :url, null: true, default: nil + + Integer :story_count, null: false, default: 0 + Boolean :completed, null: false, default: false + + DateTime :last_refresh, null: true, default: nil + + # JSON + String :data, null: false, text: true, default: '{}' + end + + create_table(:collection_stories) do + primary_key :id + + foreign_key :collection_id, :collections, on_delete: :cascade, on_update: :cascade + foreign_key :story_id, :stories, on_delete: :cascade, on_update: :cascade + Integer :index, null: true, default: nil + end + + create_table(:stories) do + primary_key :id + String :slug, null: false + String :backend_name, null: false + index %i[slug backend_name], unique: true + + foreign_key :author_id, :authors, on_delete: :cascade, on_update: :cascade + + String :name, null: false + String :synopsis, null: false, text: true + String :language, null: true, default: 'en' + + String :url, null: true, default: nil + String :image, null: true, default: nil + + Integer :chapter_count, null: false, default: 0 + Integer :word_count, null: false, default: 0 + Boolean :completed, null: false, default: false + + DateTime :published_at, null: false, default: Sequel::CURRENT_TIMESTAMP + DateTime :updated_at, null: false, default: Sequel::CURRENT_TIMESTAMP + + DateTime :last_metadata_refresh, null: true, default: nil + DateTime :last_content_refresh, null: true, default: nil + DateTime :last_accessed, null: false, default: Sequel::CURRENT_TIMESTAMP + + # JSON + String :tags, null: false, text: true, default: '[]' + String :data, null: false, text: true, default: '{}' + end + + if false + create_table(:users) do + primary_key :id + + String :username, null: false + String :email, null: true, default: nil + String :mxid, null: true, default: nil + + # JSON + String :data, null: false, text: true, default: '{}' + end + + create_table(:user_backend_auth) do + foreign_key :user_id, :users, on_delete: :cascade, on_update: :cascade + String :backend_name, null: false + + # JSON + String :authdata, null: false, text: true, default: '{}' + end + + create_table(:user_history) do + primary_key :id + + foreign_key :user_id, :users, on_delete: :cascade, on_update: :cascade + String :slug, null: false + String :backend_name, null: false + index %i[user_id slug backend_name], unique: true + + Integer :chapter_index, null: false, default: 0 + # XPATH selector for furthest page element that was reached + String :chapter_cursor, null: true, default: nil + + # JSON + String :data, null: true, text: true, default: '{}' + end + + create_table(:user_sessions) do + primary_key :id + + foreign_key :user_id, :users, on_delete: :cascade, on_update: :cascade + String :auth_token, null: false + String :refresh_token, null: false + + DateTime :session_lifetime, null: true, default: nil + DateTime :last_sync, null: false, default: Sequel::CURRENT_TIMESTAMP + + # JSON + String :data, null: false, text: true, default: '{}' + end + + create_table(:user_tracked) do + primary_key :id + + foreign_key :user_id, :users, on_delete: :cascade, on_update: :cascade + foreign_key :story_id, :stories, on_delete: :cascade, on_update: :cascade + index %i[user_id story_id], unique: true + + Integer :chapter_index, null: false, default: 0 + # XPATH selector for furthest page element that was reached + String :chapter_cursor, null: true, default: nil + + # JSON + String :data, null: false, text: true, default: '{}' + end + end + end +end + diff --git a/lib/fic_tracker/models/author.rb b/lib/fic_tracker/models/author.rb new file mode 100644 index 0000000..77d5379 --- /dev/null +++ b/lib/fic_tracker/models/author.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module FicTracker::Models + class Author < Sequel::Model + # 1/week + METADATA_REFRESH_INTERVAL = 7 * 24 * 60 * 60 + + plugin :serialization, :json, :data + + one_to_many :stories + + def before_create + backend.load_author(self) if last_metadata_refresh.nil? + + super + end + + def backend + return unless backend_name + + @backend ||= FicTracker::Backends.get(backend_name) + end + + def backend=(backend) + @backend = backend + self.backend_name = backend.name + end + + def refresh_metadata + return unless backend && needs_metadata_refresh? + + refresh_metadata! + end + + def refresh_metadata! + backend.load_author(self) + end + + def needs_metadata_refresh? + Time.now - (last_metadata_refresh || Time.at(0)) >= METADATA_REFRESH_INTERVAL + end + + def to_s + name || slug + end + end +end + diff --git a/lib/fic_tracker/models/chapter.rb b/lib/fic_tracker/models/chapter.rb new file mode 100644 index 0000000..93375f9 --- /dev/null +++ b/lib/fic_tracker/models/chapter.rb @@ -0,0 +1,98 @@ +# frozen_string_literal: true + +require 'digest' + +module FicTracker::Models + class Chapter < Sequel::Model + # 3/day + CONTENT_REFRESH_INTERVAL = 12 * 60 * 60 + CONTENT_CACHE_TIME = 7 * 24 * 60 * 60 + + plugin :serialization, :json, :data + + many_to_one :story + + def to_s + return "Chapter #{index}" unless name + + "Chapter #{index}: #{name}" + end + + def backend + story&.backend + end + + def content? + return true if @content + return false unless cache_key + + key = cache_key + [:content] + FicTracker.cache.has?(key) + end + + def content + return @content if @content + return unless cache_key + + key = cache_key + [:content] + refresh_content! unless FicTracker.cache.has?(key) + + @content ||= FicTracker.cache.get(key) + end + + def content=(content) + if cache_key + key = cache_key + [:content] + FicTracker.cache.set(key, content, CONTENT_CACHE_TIME) + end + @content = content + etag = Digest::SHA1.hexdigest(content) + end + + def content_type? + return true if @content_type + return false unless cache_key + + key = cache_key + [:content] + FicTracker.cache.has?(key) + end + + def content_type + return @content_type if @content_type + return unless cache_key + + key = cache_key + [:content_type] + refresh_content! unless FicTracker.cache.has?(key) + + @content_type ||= FicTracker.cache.get(key) + end + + def content_type=(type) + if cache_key + key = cache_key + [:content_type] + FicTracker.cache.set(key, type, CONTENT_CACHE_TIME) + end + @content_type = type + end + + def cache_key + return unless backend + + backend.cache_key + [:c, slug] + end + + def refresh_content + return unless backend && needs_content_refresh? + + refresh_content! + end + + def refresh_content! + backend.load_chapter(self, story) + end + + def needs_content_refresh? + Time.now - (last_refresh || Time.at(0)) >= CONTENT_REFRESH_INTERVAL + end + end +end diff --git a/lib/fic_tracker/models/light/search_info.rb b/lib/fic_tracker/models/light/search_info.rb new file mode 100644 index 0000000..78be963 --- /dev/null +++ b/lib/fic_tracker/models/light/search_info.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + + +module FicTracker::Models::Light + class SearchInfo + def with_tag(tag) + self + end + + def without_tag(tag) + self + end + + def with_words(count) + self + end + + class << self + protected + + def tag_category(category, **params) + end + + def word_limits(*limits) + end + + private + end + end +end diff --git a/lib/fic_tracker/models/light/tag.rb b/lib/fic_tracker/models/light/tag.rb new file mode 100644 index 0000000..bd07bfc --- /dev/null +++ b/lib/fic_tracker/models/light/tag.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module FicTracker::Models::Light + class Tag + def self.load(data) + return data if data.is_a?(Tag) || (data.is_a?(Array) && data.all? { |d| d.is_a? Tag }) + return data.map { |d| load(d) } if data.is_a?(Array) && data.all? { |d| d.is_a? Hash } + return new(**data) if data.is_a? Hash + + JSON.parse(data, symbolize_names: true).map { |obj| new(**obj) } + end + + def self.store(data) + JSON.generate(data) + end + + attr_accessor :id, :name, :category + attr_writer :ordering, :important + + def initialize(id: nil, name: nil, category: nil, important: nil, ordering: nil, **params) + @id = id + @name = name + @category = category&.to_sym + @important = important + @ordering = ordering + end + + def important? + @important + end + + def ordering + @ordering || 0 + end + + def update(**attrs) + attrs.each do |k, v| + next unless respond_to(:"#{k}=") + + send ":#{k}=", v + end + end + + def to_s + "#{category}: #{name}" + end + + def to_json(*opts) + { + id: @id, + name: @name, + category: @category, + important: @important, + ordering: @ordering, + }.compact.to_json(*opts) + end + end +end diff --git a/lib/fic_tracker/models/story.rb b/lib/fic_tracker/models/story.rb new file mode 100644 index 0000000..0541a6b --- /dev/null +++ b/lib/fic_tracker/models/story.rb @@ -0,0 +1,187 @@ +# frozen_string_literal: true + +require_relative 'light/tag' + +module FicTracker::Models + class Story < Sequel::Model + # 1/week + METADATA_REFRESH_INTERVAL = 7 * 24 * 60 * 60 + # 3/day + CONTENT_REFRESH_INTERVAL = 12 * 60 * 60 + + many_to_one :author + one_to_many :chapters, order: :index + many_to_many :collection, join_table: :collection_stories + + plugin :serialization, [Light::Tag.method(:store), Light::Tag.method(:load)], :tags + plugin :serialization, :json, :data + + # Defer creation of author/chapters until story requiring them is to be saved + def before_create + if @author + @author.save unless @author.id + self.author_id = @author.id + end + + super + end + + def after_create + return if [@author, @chapters].all?(&:nil?) + + self.author = @author if @author + @author = nil + + if @chapters + latest_chapter_at = self.updated_at || self.published_at || Time.at(0) + @chapters&.each do |chapter| + latest_chapter_at = [chapter.published_at || Time.at(0), latest_chapter_at].max + add_chapter chapter + end + update(updated_at: latest_chapter_at) if latest_chapter_at > Time.at(0) && (updated_at.nil? || latest_chapter_at >= updated_at) + end + @chapters = nil + end + + def completed? + completed + end + + # Support attaching author to a not-yet-saved story + def author + @author || super + end + + def author=(author_name) + author = author_name if author_name.is_a?(FicTracker::Models::Author) + author ||= Author.find(backend_name: backend.name, slug: author_name) + author ||= backend.load_author(author_name) if id + author ||= Author.new(backend: backend, slug: author_name) + + if id + @author = nil + author.save unless author.id + super(author) + else + @author = author + end + end + + # Support attaching chapters to a not-yet-saved story + def chapters + @chapters || super + end + + def chapters=(entries) + latest_chapter_at = self.published_at || Time.at(0) + + to_add = [] + to_remove = self.chapters.map(&:id) + + entries.each do |entry| + chapter = self.chapters.find { |c| c.slug == entry[:slug] } + + if chapter + latest_chapter_at = [chapter.published_at || Time.at(0), latest_chapter_at].max + chapter.set(**entry) + else + chapter = FicTracker::Models::Chapter.new(**entry) + to_add << chapter + end + to_remove.delete chapter.id + end + + if id + @chapters = nil + to_add.each do |entry| + logger.debug "Adding new chapter #{entry.inspect} to story #{self}" + add_chapter entry + end + if to_remove.any? + logger.debug "Removing chapter(s) #{to_remove.inspect} from story #{self}" + chapter_dataset.where(id: to_remove).delete + end + + update(updated_at: latest_chapter_at) if latest_chapter_at > Time.at(0) && (updated_at.nil? || latest_chapter_at >= updated_at) + else + @chapters = (@chapters || []) + to_add - to_remove + end + end + + def ensure_fully_loaded + ensure_chapters + refresh_content + refresh_metadata + end + + def ensure_chapters + # FIXME: Should check for a reasonable set of parameters - full load unless XX% (75%?) of chapters have content + return if chapters && chapters.any? && chapters.all? { |c| c.content? && c.content_type? } + + backend.load_full_story(self) + end + + def backend + return unless backend_name + + @backend ||= FicTracker::Backends.get(backend_name) + end + + def backend=(backend) + @backend = backend + self.backend_name = backend.name + end + + def etag + chapters.select { |c| c.etag }.last + end + + def cache_key + backend.cache_key + [:s, slug] + end + + def safe_name + [backend_name, slug, name].join('_').downcase.gsub(/[^a-z0-9\-_]/, '_').gsub(/__+/, '_') + end + + def refresh_metadata + refresh_metadata! if backend && needs_metadata_refresh? + end + + def refresh_metadata! + backend.load_story(self) + end + + def refresh_content + backend.find_chapters(self) if backend && needs_content_refresh? + chapters.each(&:refresh_content) + end + + def refresh_content! + backend.find_chapters(self) + chapters.each(&:refresh_content!) + end + + def needs_metadata_refresh? + Time.now - (last_metadata_refresh || Time.at(0)) >= METADATA_REFRESH_INTERVAL + end + + def needs_content_refresh? + Time.now - (last_content_refresh || Time.at(0)) >= (completed? ? METADATA_REFRESH_INTERVAL : CONTENT_REFRESH_INTERVAL) + end + + def to_s + "#{name}, by #{author.nil? ? '' : author.to_s}" + end + + def uid + Digest::SHA1.hexdigest("#{backend}/#{slug}") + end + + private + + def logger + Logging.logger[self] + end + end +end diff --git a/lib/fic_tracker/models/user.rb b/lib/fic_tracker/models/user.rb new file mode 100644 index 0000000..07a818d --- /dev/null +++ b/lib/fic_tracker/models/user.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module FicTracker::Models + class User < Sequel::Model + plugin :serialization, :json, :data + + one_to_many :sessions + one_to_many :tracked + one_to_many :history + + end +end diff --git a/lib/fic_tracker/renderers/epub.rb b/lib/fic_tracker/renderers/epub.rb new file mode 100644 index 0000000..e1552fd --- /dev/null +++ b/lib/fic_tracker/renderers/epub.rb @@ -0,0 +1,200 @@ +# frozen_string_literal: true + +module FicTracker::Renderers + class Epub + attr_accessor :cover, :preface + attr_reader :story, :html + + def initialize(story, io:, cover: true, preface: true) + @story = story + @io = io + + @cover = cover && story.image + @preface = preface + + @html = HTML.new(story, io: nil, body_only: true) + @html.cover = 'images/cover.jpg' if @cover + end + + def render + logger.info "Rendering epub for #{story}" + + require 'zip' + require 'zip/filesystem' + + Zip::File.open_buffer(@io, create: true, compression_level: Zlib::BEST_COMPRESSION) do |zip| + zip.dir.mkdir('META-INF') + zip.file.open('META-INF/container.xml', +'w') { |f| f.write build_container_xml } + + if cover + zip.dir.mkdir('images') + zip.file.open('images/cover.jpg', +'w') { |f| f.write build_cover_image } + end + + zip.file.open('mimetype', +'w') { |f| f.write 'application/epub+zip' } + zip.file.open('content.opf', +'w') { |f| f.write build_content_opf } + zip.file.open('toc.ncx', +'w') { |f| f.write build_toc_ncx } + + zip.file.open('preface.xhtml', +'w') { |f| f.write build_preface_xhtml } if preface + + story.chapters.each do |chapter| + zip.file.open("chapter#{chapter.index}.xhtml", +'w') { |f| f.write(build_chapter_xhtml(chapter)) } + end + end + end + + private + + def logger + Logging.logger[self] + end + + def build_cover_image + require 'vips' + + buf = Net::HTTP.get story.image + im = Vips::Image.thumbnail_buffer buf, 800, height: 1280, size: :down + + im.write_to_buffer '.jpg', strip: true + end + + def build_container_xml + Nokogiri::XML::Builder.new(encoding: 'UTF-8') do |xml| + xml.container version: '1.0', xmlns: 'urn:oasis:names:tc:opendocument:xmlns:container' do + xml.rootfiles do + xml.rootfile 'full-path': 'content.opf', 'media-type': 'application/oebps-package+xml' + end + end + end.to_xml(indent: 0) + end + + def build_content_opf + Nokogiri::XML::Builder.new(encoding: 'UTF-8') do |xml| + xml.package version: '2.0', xmlns: 'http://www.idpf.org/2007/opf', 'unique-identifier': 'story-url' do + xml.metadata 'xmlns:dc': 'http://purl.org/dc/elements/1.1/', 'xmlns:opf': 'http://www.idpf.org/2007/opf' do + xml['dc'].title story.name + xml['dc'].language story.language || 'en' + xml['dc'].identifier story.url || "#{story.backend}/#{story.slug}", id: 'story-url', 'opf:scheme': 'url' + if story.synopsis + require_relative '../converters/from_html' + xml['dc'].description FicTracker::Converters::FromHTML.to_plain(@story.synopsis) + end + xml['dc'].creator story.author&.to_s || '', 'opf:role': 'aut', 'opf:file-as': (story.author&.to_s || 'Unknown') + xml['dc'].publisher story.backend.full_name + xml['dc'].date (story.published_at || Time.now).to_datetime + xml['dc'].relation story.backend.url + xml['dc'].source story.url if story.url + story.tags.each do |tag| + xml['dc'].subject tag.to_s + end + xml.meta name: 'cover', content: 'coverImage' if cover + + xml.meta name: 'ft-backend', content: story.backend_name + xml.meta name: 'ft-story', content: story.slug + xml.meta name: 'ft-etag', content: story.etag + xml.meta name: 'ft-complete', content: story.completed? + end + + xml.manifest do + xml.item id: 'coverImage', properties: 'cover-image', href: 'images/cover.jpg', 'media-type': 'image/jpeg' if cover + xml.item id: 'preface', href: 'preface.xhtml', 'media-type': 'application/xhtml+xml' if preface + story.chapters.each do |chapter| + xml.item id: "chapter#{chapter.index}", href: "chapter#{chapter.index}.xhtml", 'media-type': 'application/xhtml+xml' + end + xml.item id: 'ncx', href: 'toc.ncx', 'media-type': 'application/x-dtbncx+xml' + end + + xml.spine(toc: 'ncx') do + xml.itemref idref: 'preface', linear: 'no' if preface + story.chapters.each do |chapter| + xml.itemref idref: "chapter#{chapter.index}" + end + end + end + end.to_xml(indent: 0) + end + + def build_toc_ncx + Nokogiri::XML::Builder.new(encoding: 'UTF-8') do |xml| + xml.doc.create_internal_subset( + 'ncx', + '-//NISO//DTD ncx 2005-1//EN', + 'http://www.daisy.org/z3986/2005/ncx-2005-1.dtd' + ) + xml.ncx xmlns: 'http://www.daisy.org/z3986/2005/ncx/', version: '2005-1', 'xml:lang': story.language || 'en' do + xml.head do + xml.meta name: 'dtb:uid', content: story.url || "#{story.backend}/#{story.slug}" + xml.meta name: 'dtb:depth', content: 2 + xml.meta name: 'dtb:generator', content: "FicTracker/#{FicTracker::VERSION}" + xml.meta name: 'dtb:totalPageCount', content: 0 + xml.meta name: 'dtb:maxPageNumber', content: 0 + end + + xml.docTitle do + xml.text story.name + end + xml.docAuthor do + xml.text story.author&.to_s || '' + end + + xml.navMap do + if preface + xml.navPoint id: 'chapter0', class: 'chapter', playOrder: '0' do + xml.navLabel do + xml.text 'Preface' + end + xml.content 'preface.xhtml' + end + end + story.chapters.each do |chapter| + xml.navPoint id: "chapter#{chapter.index}", class: 'chapter', playOrder: chapter.index do + xml.navLabel do + xml.text chapter.name + end + xml.content "chapter#{chapter.index}.xhtml" + end + end + end + end + end.to_xml(indent: 0) + end + + def build_preface_xhtml + Nokogiri::XML::Builder.new(encoding: 'UTF-8') do |xml| + xml.doc.create_internal_subset( + 'html', + '-//W3C//DTD XHTML 1.1//EN', + 'http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd' + ) + xml.html xmlns: 'http://www.w3.org/1999/xhtml', 'xml:lang': story.language || 'en' do + xml.head do + xml.meta 'http-equiv': 'Content-Type', content: 'application/xhtml+xml; charset=utf-8' + xml.title story.to_s + end + xml.body do + @html.build_preface(xml) + end + end + end.to_xml(indent: 0) + end + + def build_chapter_xhtml(chapter) + Nokogiri::XML::Builder.new(encoding: 'UTF-8') do |xml| + xml.doc.create_internal_subset( + 'html', + '-//W3C//DTD XHTML 1.1//EN', + 'http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd' + ) + xml.html xmlns: 'http://www.w3.org/1999/xhtml', 'xml:lang': story.language || 'en' do + xml.head do + xml.meta 'http-equiv': 'Content-Type', content: 'application/xhtml+xml; charset=utf-8' + xml.title chapter.to_s + end + xml.body do + @html.build_chapter(xml, chapter) + end + end + end.to_xml(indent: 0) + end + end +end diff --git a/lib/fic_tracker/renderers/html.rb b/lib/fic_tracker/renderers/html.rb new file mode 100644 index 0000000..516d5bd --- /dev/null +++ b/lib/fic_tracker/renderers/html.rb @@ -0,0 +1,140 @@ +# frozen_string_literal: true + +module FicTracker::Renderers + class HTML + attr_accessor :body_only, :cover, :preface + attr_reader :story + + def initialize(story, io:, cover: false, preface: true, body_only: false) + @story = story + @io = io + + @cover = story.image if cover + @preface = preface + @body_only = body_only + end + + def render + logger.info "Rendering html for #{story}" + + doc = Nokogiri::HTML5::Document.new + Nokogiri::XML::Builder.with(doc) do |html| + build_html(html) do + html.article class: "story" do + if preface + build_preface(html) + html.hr + end + + story.chapters.each do |chapter| + html.article class: 'chapter' do + build_chapter(html, chapter) + end + html.hr unless chapter == story.chapters.last + end + end + end + end + @io.puts doc.to_html + end + + def build_preface(html) + html.article class: 'preface' do + html.h1 story.name + + html.address do + html << "By " + if story.author&.url + html.a story.author.to_s, href: story.author.url + else + html.i story.author&.to_s || '' + end + end + html.a "On #{story.backend.full_name}.", href: story.url if story.url + + html.span do + html << 'Published ' + html.time story.published_at.to_date, datetime: story.published_at.iso8601 + if story.updated_at && story.updated_at != story.published_at + html << ", #{story.completed? ? 'completed' : 'updated'} " + html.time story.updated_at.to_date, datetime: story.updated_at.iso8601 + end + html << '.' + end + html.br + + html.dl do + story.tags.map { |td| FicTracker::Models::Light::Tag.load td }.sort_by(&:ordering).group_by(&:category).each do |category, tags| + html.dt category.to_s.capitalize + html.dd do + tags.each do |tag| + if tag.important? + html.b tag.name + else + html << tag.name + end + html << ', ' unless tag == tags.last + end + end + end + end + + html.article class: 'synopsis' do + html << story.synopsis + end + + html.img alt: 'Cover image', src: cover if cover + end + end + + def build_chapter(html, chapter) + html.h2 do + html.a chapter, href: chapter.url + end + + case chapter.content_type + when 'text/html' + require_relative '../converters/from_html' + html << FicTracker::Converters::FromHTML.to_safe_html(chapter.content) + when 'text/markdown' + require_relative '../converters/from_markdown' + html << FicTracker::Converters::FromMarkdown.to_safe_html(chapter.content) + when 'text/plain' + chapter.content.split("\n\n").each do |para| + html.p para + end + else + raise "Unknown chapter content-type: #{chapter.content_type.inspect}" + end + end + + private + + def logger + Logging.logger[self] + end + + def build_html(html) + return yield if body_only + + html.html(lang: story.language || 'en') do + html.head do + html.meta charset: 'utf-8' + html.meta viewport: 'width=device-width, initial-scale=1' + html.meta name: 'author', content: story.author.to_s + html.meta name: 'generator', content: "FicTracker/#{FicTracker::VERSION}" + html.meta name: 'keywords', content: story.tags.map(&:to_s).join(',') + + html.meta name: 'ft-backend', content: story.backend_name + html.meta name: 'ft-story', content: story.slug + html.meta name: 'ft-etag', content: story.etag + + html.title story.to_s + end + html.body do + yield + end + end + end + end +end diff --git a/lib/fic_tracker/renderers/markdown.rb b/lib/fic_tracker/renderers/markdown.rb new file mode 100644 index 0000000..31d4b7b --- /dev/null +++ b/lib/fic_tracker/renderers/markdown.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +module FicTracker::Renderers + class Markdown + attr_accessor :cover, :preface + attr_reader :story + + def initialize(story, io:, cover: false, preface: true) + @story = story + @io = io + + @cover = cover && story.image + @preface = preface + end + + def render + logger.info "Rendering markdown for #{story}" + + @io.puts build_preface, nil if preface + + story.chapters.each do |chapter| + @io.puts build_chapter(chapter), nil + end + + @io.puts build_chapter_references + end + + private + + def logger + Logging.logger[self] + end + + def build_preface + require_relative '../converters/from_html' + <<~EOF + #{story.name} + #{'=' * story.name.size} + + By __*[#{story.author}][aut]*__#{' '} + Published #{story.published_at.to_date}#{(story.updated_at && story.updated_at != story.published_at) ? ", updated #{story.updated_at.to_date}" : ''}.#{' '} + [The original work.](#{story.url}) + #{cover ? "![Cover image](#{story.image})\n" : ''} + #{FicTracker::Converters::FromHTML.to_md(story.synopsis)} + EOF + end + + def build_chapter(chapter) + head = "[#{chapter}][ch#{chapter.index}]\n#{'-' * (chapter.to_s.size + 6 + chapter.index.to_s.size)}\n\n" + + head + case chapter.content_type + when 'text/html' + require_relative '../converters/from_html' + FicTracker::Converters::FromHTML.to_md(chapter.content) + when 'text/markdown' + chapter.content + when 'text/plain' + chapter.content + end + end + + def build_chapter_references + refs = [] + refs << "[aut]: #{story.author.url}" if preface + refs += story.chapters.map { |c| "[ch#{c.index}]: #{c.url}" } + refs.join "\n" + end + end +end diff --git a/lib/fic_tracker/server.rb b/lib/fic_tracker/server.rb new file mode 100644 index 0000000..2eae2d4 --- /dev/null +++ b/lib/fic_tracker/server.rb @@ -0,0 +1,162 @@ +# frozen_string_literal: true + +require 'sinatra/base' + +module FicTracker + # Web server for providing your fic tracking needs + class Server < Sinatra::Base + configure :development do + require 'sinatra/reloader' + register Sinatra::Reloader + end + + configure do + root = File.join(__dir__, '../..') + + set :views, File.join(root, 'views') + set :public_folder, File.join(root, 'public') + enable :logging + end + + helpers do + def backend + return env[:ft_backend] if env[:ft_backend] + return halt 400, 'No backend provided' unless params['backend'] + + env[:ft_backend] = Backends.get params['backend'] + return halt 400, "Unable to find backend #{params['backend']}" unless env[:ft_backend] + + env[:ft_backend] + end + end + + before do + expires 3600, :public, :must_revalidate if request.request_method == 'GET' + end + + get '/', provides: :html do + haml :index, format: :html5 + end + + get '/search/:backend', provides: :html do |_backend_name| + search = backend.get_search_info + + haml :search, format: :html5, locals: { search:, backend: } + end + + get '/search/:backend', provides: :json do |_backend_name| + backend.get_search_info.to_json + end + + post '/search/:backend', provides: :html do |_backend_name| + request.body.rewind # in case someone already read it + data = JSON.parse request.body.read, symbolize_names: true + + results = backend.sarch(data) + + haml :search_result, format: :html5, locals: { results:, backend: } + end + + post '/search/:backend', provides: :json do |_backend_name| + request.body.rewind # in case someone already read it + data = JSON.parse request.body.read, symbolize_names: true + + results = backend.sarch(data) + + results.to_json + end + + get '/author/:backend/:slug', provides: :html do |_backend_name, slug| + author = Models::Author.find(backend_name: backend.name, slug:) + author ||= Models::Author.new(backend_name: backend.name, slug:) + author.refresh_metadata + + haml :author, format: :html5, locals: { author:, backend: } + ensure + author&.save_changes + end + + get '/collection/:backend/:slug', provides: :html do |_backend_name, slug| + collection = Models::Collection.find(backend_name: backend.name, slug:) + collection ||= Models::Collection.new(backend_name: backend.name, slug:) + collection.refresh_metadata + + haml :collection, format: :html5, locals: { collection:, backend: } + ensure + collection&.save_changes + end + + get '/story/:backend/:slug', provides: :html do |_backend_name, slug| + story = Models::Story.find(backend_name: backend.name, slug:) + story ||= Models::Story.new(backend_name: backend.name, slug:) + + story.ensure_fully_loaded + story.set(last_accessed: Time.now) + + last_modified story.updated_at || story.published_at + etag story.etag + + haml :story, format: :html5, locals: { story:, backend: } + ensure + story&.save_changes + end + + get '/story/:backend/:slug/:index', provides: :html do |_backend_name, slug, index| + story = Models::Story.find(backend_name: backend.name, slug:) + story ||= Models::Story.new(backend_name: backend.name, slug:) + + story.refresh_content + story.refresh_metadata + story.set(last_accessed: Time.now) + + chapter = story.chapters[index.to_i] + + last_modified chapter.published_at + etag chapter.etag + + haml :chapter, format: :html5, locals: { chapter:, story:, backend: } + ensure + story&.save_changes + end + + # rubocop:disable Metrics/BlockLength + get '/story/:backend/:slug.:format' do |_backend_name, slug, format| + render_klass = nil + mime = nil + case format + when 'epub', :epub + render_klass = Renderers::Epub + mime = 'application/epub+zip' + when 'html', :html + render_klass = Renderers::HTML + mime = 'text/html' + when 'txt', :txt, 'md', :md + render_klass = Renderers::Markdown + mime = 'text/markdown' + else + halt 400, "Unknown format #{format}" + end + + story = Models::Story.find(backend_name: backend.name, slug:) + story ||= Models::Story.new(backend_name: backend.name, slug:) + + story.refresh_content + story.refresh_metadata + story.set(last_accessed: Time.now) + + attachment "#{story.safe_name}.#{format}" + content_type mime + + last_modified story.updated_at || story.published_at + etag story.etag + + story.ensure_fully_loaded + stream do |out| + render_klass.new(story, io: out).render + end + ensure + story&.save_changes + end + # rubocop:enable Metrics/BlockLength + end +end diff --git a/lib/fic_tracker/tasks/cleanup.rb b/lib/fic_tracker/tasks/cleanup.rb new file mode 100644 index 0000000..0948683 --- /dev/null +++ b/lib/fic_tracker/tasks/cleanup.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module FicTracker::Tasks + module Cleanup + # Remove stories that haven't been accessed in a month + DELETE_UNACCESSED_STORIES = 30 * 24 * 60 * 60 + + def self.run + delete_unaccessed_stories + delete_unnecessary_authors + FicTracker.cache.expire + end + + class <1.5KB unless otherwise specified + @compress = compress || 1500 + @compress = (@compress..) if @compress.is_a? Numeric + @encoding = encoding + end + + # Expire any unwanted entries from the cache + def expire + @backend.expire if @backend.respond_to? :expire + end + + # Check if the cache contains a given key which has not expired + def has?(key) + key = expand_key(key) + return @backend.has?(key) if @backend.respond_to? :has? + + !@backend.get(key).nil? + end + # Get the value of a key, will return nil if the key is expired + def get(key) + key = expand_key(key) + decode_data @backend.get(key) + end + # Set the value for a given key, with an optional expiry + def set(key, value, expire = nil) + key = expand_key(key) + @backend.set(key, encode_data(value), expire) + end + # Get the value for a given key, or set it as the result of the block with an optional expiry + def get_or_set(key, expire = nil, &block) + key = expand_key(key) + return decode_data(@backend.get_or_set(key, expire) { encode_data(block.call) }) if @backend.respond_to? :get_or_set + return get(key) if has?(key) + + value = block.call + + @backend.set(key, encode_data(value), expire) + + value + end + # Clear the value for a given key + def delete(key) + key = expand_key(key) + @backend.delete key if @backend.respond_to? :delete + end + # Clear the entire cache + def clear + @backend.clear if @backend.respond_to? :clear + end + + private + + def logger + Logging.logger[self] + end + + attr_accessor :encoder, :compress + + ENCODING_FLAGS = 1 << 0 | 1 << 1 | 1 << 2 + COMPRESSED_FLAG = 1 << 3 + + ENCODING_NONE = 0 + ENCODING_MARSHAL = 1 + ENCODING_JSON = 2 + ENCODING_CBOR = 3 + ENCODING_CUSTOM = 4 + + def expand_key(key) + return key.join '/' if key.is_a? Array + + key + end + + # Check if a piece of data is a plain-old-object - i.e. simple data + def is_pod?(data) + return true if data.nil? || data == true || data == false || data.is_a?(String) || data.is_a?(Numeric) + return true if data.is_a?(Array) && data.all? { |k| is_pod?(k) } + return true if data.is_a?(Hash && data.all? { |k, v| k.is_a?(Symbol) && is_pod?(v) }) + + false + end + + def best_encoder?(data) + pod_encoder = :none if data.is_a?(String) + pod_encoder ||= :cbor if Object.const_defined?(:CBOR) + pod_encoder ||= :json if Object.const_defined?(:JSON) + pod_encoder ||= :marshal + return pod_encoder if is_pod?(data) + + nil + end + + def encode_data(data, encode: nil) + encode ||= @encoder + encode ||= best_encoder?(data) + + flags = 0 + case encode + when :none + data = data + flags |= ENCODING_NONE + when :marshal + data = Marshal.dump(data) + flags |= ENCODING_MARSHAL + when :json + data = data.to_json + flags |= ENCODING_JSON + when :cbor + data = data.to_cbor + flags |= ENCODING_CBOR + else + raise "Unknown encoder #{(encode || @encoder).inspect}" unless encode.nil? && @encoder.respond_to?(:dump) + + @encode.dump(data) + flags |= ENCODING_CUSTOM + end + + if @compress && @compress.include?(data.bytesize) + require 'zlib' + data = Zlib::Deflate.deflate(data, Zlib::BEST_COMPRESSION) + flags |= COMPRESSED_FLAG + end + + data.dup.prepend([flags].pack('C')) + end + + def decode_data(data) + flags = data[0].unpack('C').first + data = data[1..] + + if (flags & COMPRESSED_FLAG) == COMPRESSED_FLAG + require 'zlib' + data = Zlib::Inflate.inflate(data) + data.force_encoding('UTF-8') + end + + case (flags & ENCODING_FLAGS) + when ENCODING_MARSHAL + Marshal.load(data) + when ENCODING_JSON + JSON.parse(data, symbolize_names: true) + when ENCODING_CBOR + CBOR.decode(data) + when ENCODING_CUSTOM + @encode.load(data) + else + data + end + end + end +end diff --git a/lib/fic_tracker/util/cache/base.rb b/lib/fic_tracker/util/cache/base.rb new file mode 100644 index 0000000..2d382ba --- /dev/null +++ b/lib/fic_tracker/util/cache/base.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module FicTracker::Util::CacheImpl + class Base + def has?(key); end + def get(key); end + def set(key, value, ttl); end + def delete(key); end + end +end + diff --git a/lib/fic_tracker/util/cache/database.rb b/lib/fic_tracker/util/cache/database.rb new file mode 100644 index 0000000..1a2ca1e --- /dev/null +++ b/lib/fic_tracker/util/cache/database.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +module FicTracker::Util::CacheImpl + # A cache using live redis data + class Database < Base + attr_reader :dataset, :dataset_expired, :dataset_live + + def initialize(table: 'cache', **redis) + @dataset = FicTracker.database[table.to_s.to_sym] + @dataset_expired = @dataset.exclude{ Sequel.|({ expire_at: nil }, Sequel::CURRENT_DATE < expire_at) } + @dataset_live = @dataset.where{ Sequel.|({ expire_at: nil }, Sequel::CURRENT_DATE < expire_at) } + end + + def expire + dataset_expired.delete + end + + def has?(key) + dataset_live.filter(key: key).any? + end + + def get(key) + blob = dataset_live.filter(key: key).first&.[](:value) + return unless blob + + blob.to_s + end + + def set(key, value, expire = nil) + expire = expire >= 0 ? Time.now + expire : nil if expire.is_a?(Numeric) + expire = nil if expire.is_a?(Numeric) + + blob = Sequel.blob(value) + dataset + .insert_conflict(target: :key, update: { value: blob, expire_at: expire }) + .insert(key: key, value: blob, expire_at: expire) + + value + end + + def delete(key) + dataset.where(key: key).delete + nil + end + + def clear + dataset.delete + end + + private + + def namespace_key(key) + "#{namespace}-#{key}" + end + + def unnamespace_key(key) + key.delete_prefix("#{@namespace}-") + end + end +end diff --git a/lib/fic_tracker/util/cache/dummy.rb b/lib/fic_tracker/util/cache/dummy.rb new file mode 100644 index 0000000..3d2a83d --- /dev/null +++ b/lib/fic_tracker/util/cache/dummy.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +module FicTracker::Util::CacheImpl + # A dummy cache that doesn't store any data + class Dummy < Base + end +end diff --git a/lib/fic_tracker/util/cache/file.rb b/lib/fic_tracker/util/cache/file.rb new file mode 100644 index 0000000..ea13ed8 --- /dev/null +++ b/lib/fic_tracker/util/cache/file.rb @@ -0,0 +1,99 @@ +# frozen_string_literal: true + +require 'fileutils' +require 'tmpdir' + +module FicTracker::Util::CacheImpl + # A filesystem-backed in-memory cache + class File < Base + def initialize(dir: nil) + @dir = dir || Dir.mktmpdir('ficagg') + + @internal = Memory.new + end + + def expire + @internal.lock do + Dir[File.join(@dir, '**')].each_child do |entry| + next unless entry.end_with?('.meta') + + meta = JSON.parse(File.read(entry), symbolize_names: true) + unset meta[:key] unless meta_valid?(meta) + end + end + end + + def has?(key) + internal_has = @internal.has?(key) + return internal_has if internal_has + + File.exist?(key_path(key)) && key_valid?(key) + end + + def get(key) + # TODO: Extract the metadata ttl into the memory cache + @internal.get_or_set(key, expiry: Time.at(0)) do + File.read(key_path(key)) if has?(key) + end + end + + def set(key, value, expiry = nil) + expiry = expiry >= 0 ? Time.now + expiry : nil if expiry.is_a?(Numeric) + expiry = nil if expiry.is_a?(Numeric) + meta = { + key: key, + ttl: expiry&.to_i + }.compact.to_json + + @internal.unset(key) + @internal.get_or_set(key, expiry: expiry) do + File.write(key_path(key), value) + File.write(meta_path(key), meta) + value + end + end + + def delete(key) + @internal.unset(key) + @internal.lock do + [key_path(key), meta_path(key)].each do |path| + File.delete(path) if File.exist?(path) + end + end + end + + def clear + @internal.clear + @internal.lock do + Dir[File.join(@dir, '**')].each_child { |entry| File.delete(File.join(@dir, entry)) } + end + end + + private + + def key_path(key) + File.join(@dir, safe_key(key)) + end + def meta_path(key) + File.join(@dir, "#{safe_key(key)}.meta") + end + + def key_valid?(key) + path = meta_path(key) + return unless File.exist?(path) + + meta = JSON.parse(File.read(path), symbolize_names: true) + meta_valid?(meta) + end + + def meta_valid?(meta) + return true unless meta[:ttl] + + Time.now < Time.at(meta[:ttl]) + end + + def safe_key(key) + key.gsub(%r{[^A-Za-z0-9-]}, '_') + end + end +end diff --git a/lib/fic_tracker/util/cache/memory.rb b/lib/fic_tracker/util/cache/memory.rb new file mode 100644 index 0000000..9cb3459 --- /dev/null +++ b/lib/fic_tracker/util/cache/memory.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +module FicTracker::Util::CacheImpl + # A threadsafe in-memory cache + class Memory < Base + Entry = Struct.new('Entry', :data, :ttl) do + def valid? + return true unless ttl + + Time.now < ttl + end + + def expired? + return false unless ttl + + Time.now >= ttl + end + end + + def initialize + @data = {} + @mutex = Thread::Mutex.new + end + + def lock(&block) + @mutex.synchronize { block.call } + end + + def expire + @mutex.synchronize do + @data.delete_if { |_, v| v.expired? } + end + nil + end + + def has?(key) + @mutex.synchronize do + @data[key]&.valid? + end + end + + def get(key) + @mutex.synchronize do + @data[key].data if has?(key) + end + end + + def set(key, value, expiry = nil) + @mutex.synchronize do + expiry = expiry >= 0 ? Time.now + expiry : nil if expiry.is_a?(Numeric) + expiry = nil if expiry.is_a?(Numeric) + + @data[key] = Entry.new(value, expiry) + value + end + end + + def get_or_set(key, expiry: -1, &block) + @mutex.synchronize do + return @data[key].data if has?(key) + + value = yield + expiry = expiry >= 0 ? Time.now + expiry : nil if expiry.is_a?(Numeric) + expiry = nil if expiry.is_a?(Numeric) + + @data[key] = Entry.new(value, expiry) + value + end + end + + def delete(key) + @mutex.synchronize do + @data.delete key + end + nil + end + + def clear + @mutex.synchronize do + @data.clear + end + nil + end + end +end diff --git a/lib/fic_tracker/util/cache/redis.rb b/lib/fic_tracker/util/cache/redis.rb new file mode 100644 index 0000000..2d4f5ae --- /dev/null +++ b/lib/fic_tracker/util/cache/redis.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +require 'redis' + +module FicTracker::Util::CacheImpl + class Redis < Base + # A cache using live redis data + def initialize(namespace: 'cache', **redis) + @client = Redis.new(**redis) + + @namespace = namespace + end + + def has?(key) + @client.exists?(namespace_key(key)) + end + + def get(key) + @client.get(namespace_key(key)) + end + + def set(key, value, expiry = nil) + key = namespace_key(key) + + expiry = expiry >= 0 ? Time.now + expiry : nil if expiry.is_a?(Numeric) + expiry = nil if expiry.is_a?(Numeric) + + @client.set(key, value, exat: expiry) + + value + end + + def delete(key) + key = namespace_key(key) + @client.del key + nil + end + + def clear + @client.scan_each(match: "#{@namespace}-*") { |key| @client.del key } + end + + def each + @data.scan_each(match: "#{@namespace}-*") { |key| yield @client.get(key) } + end + + private + + def namespace_key(key) + "#{namespace}-#{key}" + end + + def unnamespace_key(key) + key.delete_prefix("#{@namespace}-") + end + end +end diff --git a/lib/fic_tracker/util/database.rb b/lib/fic_tracker/util/database.rb new file mode 100644 index 0000000..ec9936a --- /dev/null +++ b/lib/fic_tracker/util/database.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +require 'json' +require 'sequel' +require 'sequel/plugins/serialization' + +module FicTracker::Util + class Database + DEFAULT_SESSION_TIMEOUT = 48 * 60 * 60 + + def self.connect(connection_string, migrate: true, options: {}) + raise 'Must provide a database URL' if connection_string.nil? || connection_string.empty? + + db = if connection_string == :memory + Sequel.sqlite + else + Sequel.connect(connection_string, **options) + end + + db.sql_log_level = :debug + + # Defer logging until after migrations for in-memory store + db.loggers << Logging.logger[self] unless connection_string == :memory + self.migrate(db) if migrate + db.loggers << Logging.logger[self] if connection_string == :memory + + db + end + + def self.migrate(db) + Sequel.extension :migration + Sequel::Migrator.run(db, File.join(__dir__, '..', 'migrations')) + end + end +end diff --git a/lib/fic_tracker/util/hash_extensions.rb b/lib/fic_tracker/util/hash_extensions.rb new file mode 100644 index 0000000..6e68bcc --- /dev/null +++ b/lib/fic_tracker/util/hash_extensions.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +class Hash + def deep_transform_keys(&block) + to_h { |k, v| [block.call(k), v.is_a?(Hash) ? v.deep_transform_keys(&block) : v] } + end +end diff --git a/lib/fic_tracker/version.rb b/lib/fic_tracker/version.rb new file mode 100644 index 0000000..76bc538 --- /dev/null +++ b/lib/fic_tracker/version.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +module FicTracker + VERSION = "0.1.0" +end diff --git a/test/test_fic_tracker.rb b/test/test_fic_tracker.rb new file mode 100644 index 0000000..bdcbc99 --- /dev/null +++ b/test/test_fic_tracker.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +require "test_helper" + +class TestFicTracker < Minitest::Test + def test_that_it_has_a_version_number + refute_nil ::FicTracker::VERSION + end + + def test_it_does_something_useful + assert false + end +end diff --git a/test/test_helper.rb b/test/test_helper.rb new file mode 100644 index 0000000..88dee09 --- /dev/null +++ b/test/test_helper.rb @@ -0,0 +1,6 @@ +# frozen_string_literal: true + +$LOAD_PATH.unshift File.expand_path("../lib", __dir__) +require "fic_tracker" + +require "minitest/autorun" diff --git a/views/author.haml b/views/author.haml new file mode 100644 index 0000000..e69de29 diff --git a/views/chapter.haml b/views/chapter.haml new file mode 100644 index 0000000..e69de29 diff --git a/views/index.haml b/views/index.haml new file mode 100644 index 0000000..e69de29 diff --git a/views/story.haml b/views/story.haml new file mode 100644 index 0000000..e69de29