commit ff39289d6b2d0db31028133b91ee6d9dad23b5e2 Author: Alexander Olofsson Date: Sat May 25 23:31:26 2024 +0200 Initial commit diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..7fb6379 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,27 @@ +name: Ruby + +on: + push: + branches: + - master + + pull_request: + +jobs: + build: + runs-on: ubuntu-latest + name: Ruby ${{ matrix.ruby }} + strategy: + matrix: + ruby: + - '3.2.3' + + steps: + - uses: actions/checkout@v4 + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby }} + bundler-cache: true + - name: Run the default task + run: bundle exec rake diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ebe157b --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +/.bundle/ +/.yardoc +/_yardoc/ +/coverage/ +/doc/ +/pkg/ +/spec/reports/ +/tmp/ +/vendor/ +/Gemfile.lock +/database.db +/config.yml diff --git a/.rubocop.yml b/.rubocop.yml new file mode 100644 index 0000000..1a44467 --- /dev/null +++ b/.rubocop.yml @@ -0,0 +1,13 @@ +AllCops: + TargetRubyVersion: 3.2 + +Style/StringLiterals: + Enabled: true + # EnforcedStyle: single_quotes + +Style/StringLiteralsInInterpolation: + Enabled: true + # EnforcedStyle: single_quotes + +Layout/LineLength: + Max: 120 diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..b68d1e6 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,5 @@ +## [Unreleased] + +## [0.1.0] - 2024-04-25 + +- Initial release diff --git a/Gemfile b/Gemfile new file mode 100644 index 0000000..3f8b1cc --- /dev/null +++ b/Gemfile @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +# Specify your gem's dependencies in fic_tracker.gemspec +gemspec + +gem "rake", "~> 13.0" + +gem "minitest", "~> 5.16" + +gem "rubocop", "~> 1.21" diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..6c9975e --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2024 Alexander Olofsson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..c3cc100 --- /dev/null +++ b/README.md @@ -0,0 +1,16 @@ +# FicTracker + +A web application to handle tracking and reading live fiction, as it's updated. + + +## Usage + +Instantiate config.yml and use rackup + +## Contributing + +Bug reports and pull requests are welcome on GitHub at https://github.com/ananace/ruby-fic_tracker. + +## License + +The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT). diff --git a/Rakefile b/Rakefile new file mode 100644 index 0000000..2bf771f --- /dev/null +++ b/Rakefile @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +require "bundler/gem_tasks" +require "minitest/test_task" + +Minitest::TestTask.create + +require "rubocop/rake_task" + +RuboCop::RakeTask.new + +task default: %i[test rubocop] diff --git a/bin/fic_tracker b/bin/fic_tracker new file mode 100755 index 0000000..4bf9493 --- /dev/null +++ b/bin/fic_tracker @@ -0,0 +1,73 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require 'fic_tracker' +require 'optparse' +require 'ostruct' + +options = OpenStruct.new +OptParse.new do |opts| + opts.banner = 'Usage: fic_tracker [OPTIONS...]' + + opts.on '-b', '--backend=BACKEND', 'The backend to use' do |backend| + options.backend = backend + end + + opts.on '-s', '--story=STORY', 'The story to download' do |story| + options.story = story + end + + opts.on '-c', '--chapter=CHAPTER', 'The chapter to download' do |chapter| + options.chapter = chapter + end + + opts.on '-f', '--format=FORMAT', 'The format to download (epub, html, markdown)' do |format| + options.format = format.to_sym + end + + opts.on '-o', '--output=FILE', 'The resulting file to save the download into' do |output| + options.output = output + end + + opts.separator '' + + opts.on '-C', '--config=FILE', 'Specify a configuration file to read' do |config| + options.config = config + end + + opts.on '-h', '--help', 'Show this text' do + puts opts + exit + end + + opts.on '-q', '--quiet', 'Run quietly' do + options.log_level = :error + end + + opts.on '-v', '--verbose', 'Run verbosely, can be specified twice for debug output' do + if %i[info debug].include?(options.log_level) + options.log_level = :debug + else + options.log_level = :info + end + end + + opts.on '-V', '--version', 'Prints the version and exits' do + puts FicTracker::VERSION + exit + end +end.parse! + +FicTracker.logger.level = options.log_level || :warn +FicTracker.configure! + +backend = FicTracker::Backends.get(options.backend) +story = FicTracker::Models::Story.find(backend_name: backend.name, slug: backend.parse_slug(options.story)) || FicTracker::Models::Story.new(backend: backend, slug: backend.parse_slug(options.story)) +story.ensure_fully_loaded +data = nil + +options.output ||= "#{story.safe_name}.#{options.format}" +FicTracker.logger.info "Saving to #{options.output}" +File.open(options.output, 'w') { |f| FicTracker::Renderers.render(options.format, story, io: f) } + +story.save_changes diff --git a/config.ru b/config.ru new file mode 100644 index 0000000..732d2ac --- /dev/null +++ b/config.ru @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'fic_tracker' + +FicTracker.configure! + +map '/' ->() { run FicTracker::Server } diff --git a/config.yml.example b/config.yml.example new file mode 100644 index 0000000..97328e3 --- /dev/null +++ b/config.yml.example @@ -0,0 +1,6 @@ +--- +database: + url: sqlite://database.db + +cache: + type: database diff --git a/fic_tracker.gemspec b/fic_tracker.gemspec new file mode 100644 index 0000000..9b5b60a --- /dev/null +++ b/fic_tracker.gemspec @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +require_relative "lib/fic_tracker/version" + +Gem::Specification.new do |spec| + spec.name = "fic_tracker" + spec.version = FicTracker::VERSION + spec.authors = ["Alexander Olofsson"] + spec.email = ["ace@haxalot.com"] + + spec.summary = 'A tracker and aggregator for live fiction' + spec.description = 'A web service that tracks and aggregates live fiction for use with e-readers or similar devices' + spec.homepage = 'https://github.com/ananace/ruby-fic_tracker' + spec.license = 'MIT' + spec.required_ruby_version = '>= 3.0.0' + + spec.metadata["allowed_push_host"] = '' + + spec.metadata["homepage_uri"] = spec.homepage + spec.metadata["source_code_uri"] = spec.homepage + spec.metadata["changelog_uri"] = "#{spec.homepage}/blob/master/CHANGELOG.md" + + spec.files = Dir['{lib,views,public}/**/*'] + %w[CHANGELOG.md LICENSE.txt README.md config.ru config.yml.example fic_tracker.gemspec] + + spec.add_dependency 'haml' + spec.add_dependency 'kramdown' + spec.add_dependency 'logging' + spec.add_dependency 'nokogiri' + spec.add_dependency 'rackup' + spec.add_dependency 'ruby-vips' + spec.add_dependency 'rubyzip' + spec.add_dependency 'sequel' + spec.add_dependency 'sinatra' + spec.add_dependency 'sqlite3' +end diff --git a/lib/fic_tracker.rb b/lib/fic_tracker.rb new file mode 100644 index 0000000..4a61063 --- /dev/null +++ b/lib/fic_tracker.rb @@ -0,0 +1,101 @@ +# frozen_string_literal: true + +require 'logging' + +require_relative 'fic_tracker/version' + +module FicTracker + class Error < StandardError; end + + class ConfigError < Error; end + + def self.logger + @logger ||= Logging.logger[self].tap do |logger| + logger.add_appenders ::Logging.appenders.stdout + logger.level = :warn + end + end + + def self.configure! + Config.load! + end + + def self.cache + configure! unless @cache + @cache + end + + def self.cache=(cache) + @cache = cache + end + + def self.database + configure! unless @database + @database + end + + def self.database=(database) + @database = database + end + + def self.debug! + logger.level = :debug + end + + def self.logger=(global_logger) + @logger = global_logger + @global_logger = !global_logger.nil? + end + + def self.global_logger? + @global_logger ||= false + end + + module Models + def self.const_missing(const) + raise 'No database connected' unless FicTracker.database + + model = const.to_s.downcase + require_relative "fic_tracker/models/#{model}" + + mod = const_get(const) if const_defined? const + return mod if mod + + raise "Model not found: #{const}" + end + end + + module Tasks + autoload :Cleanup, 'fic_tracker/tasks/cleanup' + end + + module Renderers + autoload :Epub, 'fic_tracker/renderers/epub' + autoload :HTML, 'fic_tracker/renderers/html' + autoload :Markdown, 'fic_tracker/renderers/markdown' + + def self.render(type, story, **attrs) + klass = case type + when :Markdown, :markdown, :md + Markdown + when :HTML, :html + HTML + when :Epub, :epub + Epub + end + + unless attrs[:io] + require 'stringio' + attrs[:io] = StringIO.new + attrs[:_stringio] = true + end + + return klass.new(story, **attrs).render unless attrs[:stringio] + + attrs[:io].string + end + end +end + +require_relative 'fic_tracker/backend' +require_relative 'fic_tracker/config' diff --git a/lib/fic_tracker/backend.rb b/lib/fic_tracker/backend.rb new file mode 100644 index 0000000..c0e029e --- /dev/null +++ b/lib/fic_tracker/backend.rb @@ -0,0 +1,96 @@ +# frozen_string_literal: true + +module FicTracker + class Backend + class SearchInfo + attr_accessor :tag_categories + end + + def self.backend_name + name.split('::')[-2] + end + + def self.config_name + backend_name.downcase.to_sym + end + + def name + self.class.backend_name + end + + def full_name + self.class.name + end + + def url; end + + def load_author(author); end + def load_story(slug); end + def find_chapters(story); end + def load_full_story(story) + story = load_story(story) + find_chapters(story) + story.chapters.each { |c| c.refresh_content } + story + end + def load_chapter(slug, story); end + + def get_search_info; end + def search(search_info); end + + def parse_slug(slug) + slug + end + + def cache_key + [ self.class.name.split('::').map(&:downcase)[-2, 1] ] + end + + protected + + def logger + Logging.logger[self] + end + + def cache + FicTracker.cache + end + end + + module Backends + def self.get(name) + const = name + const = const.to_s.to_sym unless const.is_a? Symbol + + model = constants.find { |c| c == const } + return const_get(model).const_get(:Backend).new if model + + model = load(name) + return model if model + + raise "Unknown backend #{name}" + end + + def self.const_missing(const) + mod = load(const) + return mod if mod + + raise "Unknown backend #{const}" + end + + def self.load(name) + model = name.to_s.downcase + require_relative "backends/#{model}/backend" + + model = constants.find { |c| c.to_s.downcase == model } + return unless model + + backend = const_get(model).const_get(:Backend) + backend.new(**FicTracker.config.dig(:backends, backend.config_name, default: {})) + rescue StandardError => e + Logging.logger[backend].error "Failed to load, #{e.class}: #{e}" + + nil + end + end +end diff --git a/lib/fic_tracker/backends/ao3/backend.rb b/lib/fic_tracker/backends/ao3/backend.rb new file mode 100644 index 0000000..7a8c389 --- /dev/null +++ b/lib/fic_tracker/backends/ao3/backend.rb @@ -0,0 +1,204 @@ +# frozen_string_literal: true + +require_relative 'client' + +module FicTracker::Backends::Ao3 + class Backend < FicTracker::Backend + TAG_ORDERING = { + rating: -3, + category: -2, + warning: -1, + }.freeze + + def client + @client ||= Client.new + end + + def full_name + 'Archive of Our Own' + end + + def url + Client::BASE_URL + end + + def load_author(author) + author = FicTracker::Models::Author.new(slug: parse_slug(author), backend: self) unless author.is_a? FicTracker::Models::Author + + logger.info "Loading author #{author.slug}" + doc = client.request("/users/#{author.slug}") + + user = doc.at_css('#main .user') + + url = user.at_css('.icon a') + url = URI.join(Client::BASE_URL, url[:href]) if url + image = user.at_css('.icon a img') + image = URI.join(Client::BASE_URL, image[:src]) if image + + author.set( + name: user.at_css('h2.heading').text.strip, + url: url.to_s, + image: image.to_s, + last_metadata_refresh: Time.now + ) + end + + def load_story(story) + story = FicTracker::Models::Story.new(slug: parse_slug(story), backend: self) unless story.is_a? FicTracker::Models::Story + + logger.info "Loading story #{story.slug}" + doc = client.request("/works/#{story.slug}") + + attrs = extract_story doc + + story.set(last_metadata_refresh: Time.now, **attrs) + end + + def find_chapters(story) + story = FicTracker::Models::Story.new(slug: parse_slug(story), backend: self) unless story.is_a? FicTracker::Models::Story + + logger.info "Loading chapters for #{story.slug}" + doc = client.request("/works/#{story.slug}/navigate") + + chapters = doc.at_css('ol.chapter').css('li').map do |entry| + published_at = Time.parse(entry.at_css('span.datetime').text.strip) + link = entry.at_css('a') + index, *name = link.text.split('. ') + index = index.to_i + name = name.join('. ') + url = URI.join(Client::BASE_URL, link[:href]) + slug = url.path.split('/').last + + { + slug: slug, + index: index, + name: name, + url: url.to_s, + published_at: published_at, + } + end + + story.set( + last_content_refresh: Time.now, + ) + story.chapters = chapters + + story + end + + def load_full_story(story) + story = FicTracker::Models::Story.new(slug: parse_slug(story), backend: self) unless story.is_a? FicTracker::Models::Story + + logger.info "Loading all chapters for #{story.slug}" + doc = client.request("/works/#{story.slug}", query: { view_full_work: true }) + + attrs = extract_story(doc) + chapters = doc.css('#chapters > div.chapter').map { |chapter| extract_chapter(chapter) } + + story.set( + last_metadata_refresh: Time.now, + last_content_refresh: Time.now, + **attrs + ) + story.chapters = chapters + + story + end + + def load_chapter(chapter, story) + story = load_story(story) unless story.is_a? FicTracker::Models::Story + chapter = FicTracker::Models::Chapter.new(slug: parse_slug(chapter), story: story) unless chapter.is_a? FicTracker::Models::Chapter + + logger.info "Loading chapter #{chapter.slug} for #{story.slug}" + doc = client.request("/works/#{story.slug}/chapters/#{chapter.slug}") + + attrs = extract_chapter(doc.at_css('#chapters > div.chapter')) + + chapter.set(**attrs) + end + + def parse_slug(slug) + return URI(slug).path.split('/').last if slug.is_a?(String) && slug.start_with?('http') + + slug.to_s + end + + private + + def extract_story(doc) + url = URI.join(Client::BASE_URL, doc.at_css('li.share a')[:href].delete_suffix('/share')) + meta = doc.at_css('#main dl.meta') + preface = doc.at_css('#workskin .preface') + + name = preface.at_css('.title').text.strip + synopsis = preface.at_css('.summary blockquote').children.to_xml + language = meta.at_css('dd.language')['lang'] + author = preface.at_css('a[rel="author"]')[:href].split('/')[2] + + tags = meta.css('dd.tags').map do |tagblock| + category = tagblock[:class].split.first.to_sym + + tagblock.css('a.tag').map do |a| + next if ['Creator Chose Not To Use Archive Warnings', 'No Archive Warnings Apply'].include? a.text.strip + + { + name: a.text.strip, + category: category, + important: %i[rating warning category].include?(category) ? true : nil, + ordering: TAG_ORDERING[category], + }.compact + end.compact + end.flatten + + chapters = meta.at_css('dl.stats dd.chapters').text.strip.split('/') + words = meta.at_css('dl.stats dd.words').text.strip.tr(',', '').to_i + published_at = meta.at_css('dl.stats dd.published')&.text&.strip + published_at = Time.parse(published_at) if published_at + updated_at = meta.at_css('dl.stats dd.status')&.text&.strip + updated_at = Time.parse(updated_at) if updated_at + + { + name: name, + author: author, + synopsis: synopsis, + url: url.to_s, + language: language, + chapter_count: chapters.first.to_i, + word_count: words, + completed: chapters.first == chapters.last, + published_at: published_at, + updated_at: updated_at, + tags: FicTracker::Models::Light::Tag.load(tags), + } + end + + def extract_chapter(chapter_doc) + link = chapter_doc.at_css('h3.title a') + url = URI.join(Client::BASE_URL, link[:href]) + slug = url.path.split('/').last + index = link.text.strip.split.last.to_i + + html = chapter_doc.at_css('div[role="article"]').children[2..].map(&:to_xml).join("\n") + + title = chapter_doc.at_css('h3.title') + if title + title_base = title.at_css('a').text.strip + title_extra = title.text.strip.delete_prefix("#{title_base}:").strip + + title = title_extra.empty? ? title_base : title_extra + end + + { + index: index, + slug: slug, + + name: title, + url: url.to_s, + last_refresh: Time.now, + + content: html, + content_type: 'text/html', + } + end + end +end diff --git a/lib/fic_tracker/backends/ao3/client.rb b/lib/fic_tracker/backends/ao3/client.rb new file mode 100644 index 0000000..ab3928e --- /dev/null +++ b/lib/fic_tracker/backends/ao3/client.rb @@ -0,0 +1,100 @@ +# frozen_string_literal: true + +require 'cgi' +require 'net/http' +require 'nokogiri' +require 'json' + +module FicTracker::Backends::Ao3 + class Client + BASE_URL = 'https://archiveofourown.org' + + def url + URI(BASE_URL) + end + + def request(path, type: :html, body: nil, query: nil, method: :get, redirect: true) + uri = URI.join(url, path) + uri.query = URI.encode_www_form(query) if query + + req = Net::HTTP.const_get(method.to_s.capitalize.to_sym).new uri.request_uri + req['User-Agent'] = "FicTracker/#{FicTracker::VERSION}" + case type + when :html, :xml + req['Accept'] = 'text/html,application/xhtml+xml,application/xml' + when :json + req['Accept'] = 'application/json' + end + + if body + req.body = body + req.body = req.body.to_json unless req.body.is_a? String + req['Content-Type'] = 'application/json' + end + + resp = nil + http.start do + loop do + debug_http(req) + resp = http.request req + debug_http(resp) + case resp + when Net::HTTPRedirection + req.path.replace resp['location'] + when Net::HTTPTooManyRequests + wait_time = 10 + if resp['retry-after'] + after = resp['retry-after'] + wait_time = after =~ /\A[0-9]+\Z/ ? after.to_i : Time.parse(after) + end + wait_time = Time.now + wait_time if wait_time.is_a? Numeric + + logger.info "Rate limited, waiting until #{wait_time} before retrying" + sleep Time.now - wait_time + else + break + end + end + end + + resp.value + return if resp.body.empty? + + case type + when :html + Nokogiri::HTML4.parse(resp.body) + when :xml + Nokogiri::XML.parse(resp.body) + when :json + JSON.parse(resp.body, symbolize_names: true) + when :raw + resp.body + end + end + + private + + def logger + Logging.logger[self] + end + + def http + @http ||= Net::HTTP.new(url.host, url.port).tap { |http| http.use_ssl = url.scheme == 'https' } + end + + def debug_http(object) + return unless logger.debug? + + dir = '>' + if object.is_a?(Net::HTTPRequest) + dir = '<' + + logger.debug "#{dir} #{object.method} #{object.path}" + else + logger.debug "#{dir} #{object.code} #{object.message}" + end + + object.each_header { |h, v| logger.debug "#{dir} #{h}: #{v}" } + end + end +end diff --git a/lib/fic_tracker/backends/fimfiction/backend.rb b/lib/fic_tracker/backends/fimfiction/backend.rb new file mode 100644 index 0000000..6f8fdbe --- /dev/null +++ b/lib/fic_tracker/backends/fimfiction/backend.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +require_relative 'fimfiction/client' + +module FicTracker::Backends::FIMFiction + class Backend < FicTracker::Backend + def client + @client ||= Client.new + end + + def full_name + 'FIM Fiction' + end + + def url + 'https://www.fimfiction.net' + end + + def load_author(author) + author = FicTracker::Models::Author.new(slug: author, backend: self) unless author.is_a? FicTracker::Models::Author + + logger.info "Loading author #{author.slug}" + end + def load_story(slug) + story = FicTracker::Models::Story.new(slug: story, backend: self) unless story.is_a? FicTracker::Models::Story + + logger.info "Loading story #{story.slug}" + end + def find_chapters(story) + story = FicTracker::Models::Story.new(slug: story, backend: self) unless story.is_a? FicTracker::Models::Story + + logger.info "Loading chapters for #{story.slug}" + end + def load_chapter(chapter, story) + story = FicTracker::Models::Story.new(slug: story, backend: self) unless story.is_a? FicTracker::Models::Story + chapter = FicTracker::Models::Chapter.new(slug: chapter, story: story) unless chapter.is_a? FicTracker::Models::Chapter + + logger.info "Loading chapter #{chapter.slug} for #{story.slug}" + end + + def get_search_info; end + def search(search_info); end + end +end diff --git a/lib/fic_tracker/backends/fimfiction/client.rb b/lib/fic_tracker/backends/fimfiction/client.rb new file mode 100644 index 0000000..2d2fa50 --- /dev/null +++ b/lib/fic_tracker/backends/fimfiction/client.rb @@ -0,0 +1,99 @@ +# frozen_string_literal: true + +require 'cgi' +require 'json' +require 'net/http' +require 'uri' + +module FicTracker::Backends::FIMFiction + class Client + BASE_URL = URI('https://www.fimfiction.net/api/v2').freeze + + attr_accessor :access_token, :url + attr_reader :tags + + def initialize + @access_token = nil + @url = BASE_URL + + @tags = Tags.new(self) + end + + def request(path, body: nil, query: nil, method: :get) + uri = URI.join(BASE_URL, path) + uri.query = URI.encode_www_form(query) if query + + req = Net::HTTP.const_get(method.to_s.capitalize.to_sym).new + req['Accept'] = 'application/vnd.api+json' + req['Authorization'] = "Bearer #{access_token}" if access_token + req['User-Agent'] = "FicTracker/#{FicTracker::VERSION}" + + if body + req.body = body + req.body = req.body.to_json unless req.body.is_a? String + req['Content-Type'] = 'application/json' + end + + resp = nil + http.start do + loop do + debug_http(req) + resp = http.request req + debug_http(resp) + case resp + when Net::HTTPRedirection + req.path.replace resp['location'] + when Net::HTTPTooManyRequests + wait_time = 10 + if resp['retry-after'] + after = resp['retry-after'] + wait_time = after =~ /\A[0-9]+\Z/ ? after.to_i : Time.parse(after) + end + wait_time = Time.now + wait_time if wait_time.is_a? Numeric + + logger.info "Rate limited, waiting until #{wait_time} before retrying" + sleep Time.now - wait_time + else + break + end + end + end + + resp.value + return if resp.body.empty? + + return JSON.parse(resp.body, symbolize_names: true) + end + + def retrieve_token(id, secret) + data = request('/token', body: { client_id: id, client_secret: secret, grant_type: :client_credentials }, method: :post) + + @access_token = data[:access_token] + end + + private + + def logger + Logging.logger[self] + end + + def http + @http ||= Net::HTTP.new(url.host, url.port).tap { |http| http.use_ssl = url.scheme == 'https' } + end + + def debug_http(object) + return unless logger.debug? + + dir = '>' + if object.is_a?(Net::HTTPRequest) + dir = '<' + + logger.debug "#{dir} #{object.method} #{object.path}" + else + logger.debug "#{dir} #{object.code} #{object.message}" + end + + object.each_header { |h, v| logger.debug "#{dir} #{h}: #{v}" } + end + end +end diff --git a/lib/fic_tracker/config.rb b/lib/fic_tracker/config.rb new file mode 100644 index 0000000..d5de1bb --- /dev/null +++ b/lib/fic_tracker/config.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +require 'psych' + +require_relative 'util/cache' +require_relative 'util/database' + +module FicTracker::Config + class << self + def load!(config_file: 'config.yml') + @config_file = config_file + @config = {} + + load_internal + + FicTracker.database = FicTracker::Util::Database.connect( + dig(:database, :url, default: :memory), + migrate: dig(:database, :migrate, default: true), + options: dig(:database, :options, default: {}) + ) + FicTracker.cache = FicTracker::Util::Cache.create( + type: dig(:cache, :type, default: :none), + compress: dig(:cache, :compress, default: nil), + encoding: dig(:cache, :encoding, default: nil), + options: dig(:cache, :options, default: {}) + ) + + Sequel::Model.db = FicTracker.database + end + + def [](arg) + @config[arg] + end + + def dig(*args, default: nil) + envvar = (%i[ft] + args.map { |arg| arg.to_s }).join('_').upcase + return ENV[envvar] if ENV[envvar] + + ret = @config.dig(*args) + return ret if ret + return default unless default.nil? + return yield if block_given? + + nil + end + + private + + def load_internal + @config_file = ENV['FT_CONFIG_FILE'] if ENV['FT_CONFIG_FILE'] + begin + @config = Psych.load(File.read(@config_file)).deep_transform_keys(&:to_sym) + rescue + @config = {} + end + @config[:database] ||= {} + @config[:database][:config] ||= {} + @config[:cache] ||= {} + + if ENV['FT_CACHE_REDIS_URL'] + @config[:cache][:type] = :redis + @config[:cache][:url] = ENV['FT_CACHE_REDIS_URL'] + end + end + end +end diff --git a/lib/fic_tracker/converters/from_html.rb b/lib/fic_tracker/converters/from_html.rb new file mode 100644 index 0000000..759589d --- /dev/null +++ b/lib/fic_tracker/converters/from_html.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +require 'kramdown' + +module FicTracker::Converters + class FromHTML + def self.to_plain(content) + require_relative 'to_plain' + + Kramdown::Document.new(content, input: 'html').to_plain.strip + end + + def self.to_md(content) + require_relative 'to_simplemark' + + Kramdown::Document.new(content, input: 'html').to_simplemark.strip + end + + def self.to_safe_html(content) + require_relative 'to_safe_html' + + Kramdown::Document.new(content, input: 'html').to_safe_html + end + end +end diff --git a/lib/fic_tracker/converters/from_markdown.rb b/lib/fic_tracker/converters/from_markdown.rb new file mode 100644 index 0000000..90607a6 --- /dev/null +++ b/lib/fic_tracker/converters/from_markdown.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +require 'kramdown' + +module FicAggregator::Converters + class FromMarkdown + def self.to_plain(content) + require_relative 'to_plain' + + Kramdown::Document.new(content, input: 'markdown').to_plain.strip + end + + def self.to_html(content) + Kramdown::Document.new(content, input: 'markdown').to_html.strip + end + + def self.to_safe_html(content) + Kramdown::Document.new(content, input: 'markdown').to_safe_html + end + end +end diff --git a/lib/fic_tracker/converters/to_plain.rb b/lib/fic_tracker/converters/to_plain.rb new file mode 100644 index 0000000..858dd24 --- /dev/null +++ b/lib/fic_tracker/converters/to_plain.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +require_relative 'to_simplemark' + +module Kramdown::Converter + # Markdown with some slight simplification to ease reading + class Plain < Simplemark + private + + def convert_header(el, depth = 1) + depth = 7 - depth + "#{'=' * depth} #{inner(el)} #{'=' * depth}\n\n" + end + + def convert_strong(el) + "*#{inner(el)}*" + end + + def convert_em(el) + "_#{inner(el)}_" + end + + def convert_a(el) + content = inner(el) + [content, "(#{el.attr['href']})"].reject { |k| k.nil? || k.empty? }.join ' ' + end + + def convert_blockquote(el) + " #{inner(el).split("\n").join("\n ")}" + end + end +end diff --git a/lib/fic_tracker/converters/to_safe_html.rb b/lib/fic_tracker/converters/to_safe_html.rb new file mode 100644 index 0000000..2beba9c --- /dev/null +++ b/lib/fic_tracker/converters/to_safe_html.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +require 'kramdown' +require 'logging' + +module Kramdown::Converter + # HTML converter that strips out elements not relevant for e-readers + class SafeHtml < Base + def initialize(root, options) + super + + @options[:remove_block_html_tags] = true + @options[:remove_span_html_tags] = false + @options[:template] = 'string://<%= Html.convert(@body).first %>' + end + + SUPERFLUOUS_TAGS = %w[align class justify] + + def convert(el) + real_el, el = el, el.value if el.type == :footnote + + # Strip out unnecessary HTML tags + SUPERFLUOUS_TAGS.each { |tag| el.attr.delete tag if el.attr.key? tag } + + children = el.children.dup + index = 0 + + while index < children.length + if %i[img xml_pi].include?(children[index].type) || + (children[index].type == :html_element && %w[style script title].include?(children[index].value)) + children[index..index] = [] + elsif children[index].type == :html_element && ( + (@options[:remove_block_html_tags] && children[index].options[:category] == :block) || + (@options[:remove_span_html_tags] && children[index].options[:category] == :span) + ) && children[index].value != 'div' + children[index..index] = children[index].children + else + convert(children[index]) + index += 1 + end + end + el.children = children + + real_el || el + end + end +end diff --git a/lib/fic_tracker/converters/to_simplemark.rb b/lib/fic_tracker/converters/to_simplemark.rb new file mode 100644 index 0000000..cb6a4de --- /dev/null +++ b/lib/fic_tracker/converters/to_simplemark.rb @@ -0,0 +1,176 @@ +# frozen_string_literal: true + +require 'kramdown' +require 'logging' + +module Kramdown::Converter + # Simplified commonmark converter + class Simplemark < Base + def convert(el, **_) + meth = :"convert_#{el.type}" + meth = :"convert_html_#{el.value}" if el.type == :html_element + + if self.class.private_method_defined? meth + res = send(meth, el) + else + fallback = :convert_blank + fallback = :convert_html_element if el.type == :html_element + + logger.debug "Missing #{meth.inspect} function, using #{fallback}" + res = send(fallback, el) + end + res = res.dup if res.frozen? + res + end + + private + + def logger + Logging.logger[self] + end + + class UlBullet + def next + '-' + end + end + + class OlBullet + def initialize + @at = 1 + end + + def next + "#{@at}." + ensure + @at = @at.succ! + end + end + + def inner(el) + el.children.map { |el| convert(el) }.join + end + alias convert_root inner + + def convert_blank(el) + '' + end + alias convert_comment convert_blank + + def convert_p(el) + "#{inner(el)}\n\n" + end + + def convert_text(el) + el.value + end + alias convert_abbreviation convert_text + + def convert_strong(el) + "__#{inner(el).gsub('__', '\_\_')}__" + end + alias convert_html_b convert_strong + alias convert_html_strong convert_strong + + def convert_em(el) + "_#{inner(el).gsub('_', '\_')}_" + end + alias convert_html_i convert_em + alias convert_html_em convert_em + + def convert_del(el) + "~#{inner(el)}~" + end + alias convert_html_del convert_del + + def convert_codespan(el) + "`#{el.value}`" + end + alias convert_html_code convert_codespan + + def convert_blockquote(el) + "> #{inner(el).split("\n").join("\n> ")}\n\n" + end + + def convert_codeblock(el) + "```\n#{el.value}```\n\n" + end + + def convert_a(el) + content = inner(el) + "[#{content}](#{el.attr['href']})" + end + alias convert_html_a convert_a + + def convert_header(el, depth = 1) + "#{'#' * depth} #{inner(el)} #{'#' * depth}\n\n" + end + def convert_html_h1(el); convert_header(el, 1) end + def convert_html_h2(el); convert_header(el, 2) end + def convert_html_h3(el); convert_header(el, 3) end + def convert_html_h4(el); convert_header(el, 4) end + def convert_html_h5(el); convert_header(el, 5) end + def convert_html_h6(el); convert_header(el, 6) end + + def convert_br(el) + " \n" + end + + def convert_hr(el) + "#{'-' * 16}\n\n" + end + + def convert_ul(el) + @bullet = UlBullet.new + "#{inner(el)}\n" + ensure + @bullet = nil + end + alias convert_html_ul convert_ul + + def convert_ol(el) + @bullet = OlBullet.new + "#{inner(el)}\n" + ensure + @bullet = nil + end + alias convert_html_ol convert_ol + + def convert_li(el) + " #{@bullet.next} #{inner(el).strip}\n" + end + alias convert_html_li convert_li + + def convert_smart_quote(el) + SMART_QUOTES.fetch(el.value) + end + + def convert_typographic_sym(el) + TYPOGRAPHIC_SYMBOLS.fetch(el.value) + end + + def convert_entity(el) + el.value.char + end + + alias convert_html_title convert_blank + + # Strip out unknown HTML elements, only keep content + alias convert_html_element inner + + SMART_QUOTES = { + lsquo: "‘", + rsquo: "’", + ldquo: "“", + rdquo: "”" }.freeze + + TYPOGRAPHIC_SYMBOLS = { + mdash: "—", + ndash: "–", + hellip: "...", + laquo_space: "« ", + raquo_space: " »", + laquo: "«", + raquo: "»" }.freeze + end +end diff --git a/lib/fic_tracker/migrations/001_create_base.rb b/lib/fic_tracker/migrations/001_create_base.rb new file mode 100644 index 0000000..705b465 --- /dev/null +++ b/lib/fic_tracker/migrations/001_create_base.rb @@ -0,0 +1,180 @@ +# frozen_string_literal: true + +Sequel.migration do + change do + create_table(:meta) do + String :key, null: false, primary_key: true + String :value, null: true + end + + create_table(:cache) do + String :key, null: false, primary_key: true + File :value, null: true + DateTime :expire_at, null: true, default: nil + end + + create_table(:authors) do + primary_key :id + String :slug, null: false + String :backend_name, null: false + index %i[slug backend_name], unique: true + + String :name, null: false + + String :url, null: true, default: nil + String :image, null: true, default: nil + + DateTime :last_metadata_refresh, null: true, default: nil + + # JSON + String :data, null: false, text: true, default: '{}' + end + + create_table(:chapters) do + primary_key :id + + foreign_key :story_id, :stories, on_delete: :cascade, on_update: :cascade + Integer :index, null: false + String :slug, null: false + index %i[story_id index], unique: true + index %i[story_id slug], unique: true + + String :etag, null: true, default: nil + String :name, null: true, default: nil + + String :url, null: true, default: nil + + DateTime :published_at, null: true, default: nil + DateTime :updated_at, null: true, default: nil + DateTime :last_refresh, null: true, default: nil + + # JSON + String :data, null: false, text: true, default: '{}' + end + + create_table(:collections) do + primary_key :id + String :slug, null: false + String :backend_name, null: false + index %i[slug backend_name], unique: true + + String :type, null: false + String :name, null: true, default: nil + String :url, null: true, default: nil + + Integer :story_count, null: false, default: 0 + Boolean :completed, null: false, default: false + + DateTime :last_refresh, null: true, default: nil + + # JSON + String :data, null: false, text: true, default: '{}' + end + + create_table(:collection_stories) do + primary_key :id + + foreign_key :collection_id, :collections, on_delete: :cascade, on_update: :cascade + foreign_key :story_id, :stories, on_delete: :cascade, on_update: :cascade + Integer :index, null: true, default: nil + end + + create_table(:stories) do + primary_key :id + String :slug, null: false + String :backend_name, null: false + index %i[slug backend_name], unique: true + + foreign_key :author_id, :authors, on_delete: :cascade, on_update: :cascade + + String :name, null: false + String :synopsis, null: false, text: true + String :language, null: true, default: 'en' + + String :url, null: true, default: nil + String :image, null: true, default: nil + + Integer :chapter_count, null: false, default: 0 + Integer :word_count, null: false, default: 0 + Boolean :completed, null: false, default: false + + DateTime :published_at, null: false, default: Sequel::CURRENT_TIMESTAMP + DateTime :updated_at, null: false, default: Sequel::CURRENT_TIMESTAMP + + DateTime :last_metadata_refresh, null: true, default: nil + DateTime :last_content_refresh, null: true, default: nil + DateTime :last_accessed, null: false, default: Sequel::CURRENT_TIMESTAMP + + # JSON + String :tags, null: false, text: true, default: '[]' + String :data, null: false, text: true, default: '{}' + end + + if false + create_table(:users) do + primary_key :id + + String :username, null: false + String :email, null: true, default: nil + String :mxid, null: true, default: nil + + # JSON + String :data, null: false, text: true, default: '{}' + end + + create_table(:user_backend_auth) do + foreign_key :user_id, :users, on_delete: :cascade, on_update: :cascade + String :backend_name, null: false + + # JSON + String :authdata, null: false, text: true, default: '{}' + end + + create_table(:user_history) do + primary_key :id + + foreign_key :user_id, :users, on_delete: :cascade, on_update: :cascade + String :slug, null: false + String :backend_name, null: false + index %i[user_id slug backend_name], unique: true + + Integer :chapter_index, null: false, default: 0 + # XPATH selector for furthest page element that was reached + String :chapter_cursor, null: true, default: nil + + # JSON + String :data, null: true, text: true, default: '{}' + end + + create_table(:user_sessions) do + primary_key :id + + foreign_key :user_id, :users, on_delete: :cascade, on_update: :cascade + String :auth_token, null: false + String :refresh_token, null: false + + DateTime :session_lifetime, null: true, default: nil + DateTime :last_sync, null: false, default: Sequel::CURRENT_TIMESTAMP + + # JSON + String :data, null: false, text: true, default: '{}' + end + + create_table(:user_tracked) do + primary_key :id + + foreign_key :user_id, :users, on_delete: :cascade, on_update: :cascade + foreign_key :story_id, :stories, on_delete: :cascade, on_update: :cascade + index %i[user_id story_id], unique: true + + Integer :chapter_index, null: false, default: 0 + # XPATH selector for furthest page element that was reached + String :chapter_cursor, null: true, default: nil + + # JSON + String :data, null: false, text: true, default: '{}' + end + end + end +end + diff --git a/lib/fic_tracker/models/author.rb b/lib/fic_tracker/models/author.rb new file mode 100644 index 0000000..77d5379 --- /dev/null +++ b/lib/fic_tracker/models/author.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module FicTracker::Models + class Author < Sequel::Model + # 1/week + METADATA_REFRESH_INTERVAL = 7 * 24 * 60 * 60 + + plugin :serialization, :json, :data + + one_to_many :stories + + def before_create + backend.load_author(self) if last_metadata_refresh.nil? + + super + end + + def backend + return unless backend_name + + @backend ||= FicTracker::Backends.get(backend_name) + end + + def backend=(backend) + @backend = backend + self.backend_name = backend.name + end + + def refresh_metadata + return unless backend && needs_metadata_refresh? + + refresh_metadata! + end + + def refresh_metadata! + backend.load_author(self) + end + + def needs_metadata_refresh? + Time.now - (last_metadata_refresh || Time.at(0)) >= METADATA_REFRESH_INTERVAL + end + + def to_s + name || slug + end + end +end + diff --git a/lib/fic_tracker/models/chapter.rb b/lib/fic_tracker/models/chapter.rb new file mode 100644 index 0000000..93375f9 --- /dev/null +++ b/lib/fic_tracker/models/chapter.rb @@ -0,0 +1,98 @@ +# frozen_string_literal: true + +require 'digest' + +module FicTracker::Models + class Chapter < Sequel::Model + # 3/day + CONTENT_REFRESH_INTERVAL = 12 * 60 * 60 + CONTENT_CACHE_TIME = 7 * 24 * 60 * 60 + + plugin :serialization, :json, :data + + many_to_one :story + + def to_s + return "Chapter #{index}" unless name + + "Chapter #{index}: #{name}" + end + + def backend + story&.backend + end + + def content? + return true if @content + return false unless cache_key + + key = cache_key + [:content] + FicTracker.cache.has?(key) + end + + def content + return @content if @content + return unless cache_key + + key = cache_key + [:content] + refresh_content! unless FicTracker.cache.has?(key) + + @content ||= FicTracker.cache.get(key) + end + + def content=(content) + if cache_key + key = cache_key + [:content] + FicTracker.cache.set(key, content, CONTENT_CACHE_TIME) + end + @content = content + etag = Digest::SHA1.hexdigest(content) + end + + def content_type? + return true if @content_type + return false unless cache_key + + key = cache_key + [:content] + FicTracker.cache.has?(key) + end + + def content_type + return @content_type if @content_type + return unless cache_key + + key = cache_key + [:content_type] + refresh_content! unless FicTracker.cache.has?(key) + + @content_type ||= FicTracker.cache.get(key) + end + + def content_type=(type) + if cache_key + key = cache_key + [:content_type] + FicTracker.cache.set(key, type, CONTENT_CACHE_TIME) + end + @content_type = type + end + + def cache_key + return unless backend + + backend.cache_key + [:c, slug] + end + + def refresh_content + return unless backend && needs_content_refresh? + + refresh_content! + end + + def refresh_content! + backend.load_chapter(self, story) + end + + def needs_content_refresh? + Time.now - (last_refresh || Time.at(0)) >= CONTENT_REFRESH_INTERVAL + end + end +end diff --git a/lib/fic_tracker/models/light/search_info.rb b/lib/fic_tracker/models/light/search_info.rb new file mode 100644 index 0000000..78be963 --- /dev/null +++ b/lib/fic_tracker/models/light/search_info.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + + +module FicTracker::Models::Light + class SearchInfo + def with_tag(tag) + self + end + + def without_tag(tag) + self + end + + def with_words(count) + self + end + + class << self + protected + + def tag_category(category, **params) + end + + def word_limits(*limits) + end + + private + end + end +end diff --git a/lib/fic_tracker/models/light/tag.rb b/lib/fic_tracker/models/light/tag.rb new file mode 100644 index 0000000..bd07bfc --- /dev/null +++ b/lib/fic_tracker/models/light/tag.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module FicTracker::Models::Light + class Tag + def self.load(data) + return data if data.is_a?(Tag) || (data.is_a?(Array) && data.all? { |d| d.is_a? Tag }) + return data.map { |d| load(d) } if data.is_a?(Array) && data.all? { |d| d.is_a? Hash } + return new(**data) if data.is_a? Hash + + JSON.parse(data, symbolize_names: true).map { |obj| new(**obj) } + end + + def self.store(data) + JSON.generate(data) + end + + attr_accessor :id, :name, :category + attr_writer :ordering, :important + + def initialize(id: nil, name: nil, category: nil, important: nil, ordering: nil, **params) + @id = id + @name = name + @category = category&.to_sym + @important = important + @ordering = ordering + end + + def important? + @important + end + + def ordering + @ordering || 0 + end + + def update(**attrs) + attrs.each do |k, v| + next unless respond_to(:"#{k}=") + + send ":#{k}=", v + end + end + + def to_s + "#{category}: #{name}" + end + + def to_json(*opts) + { + id: @id, + name: @name, + category: @category, + important: @important, + ordering: @ordering, + }.compact.to_json(*opts) + end + end +end diff --git a/lib/fic_tracker/models/story.rb b/lib/fic_tracker/models/story.rb new file mode 100644 index 0000000..0541a6b --- /dev/null +++ b/lib/fic_tracker/models/story.rb @@ -0,0 +1,187 @@ +# frozen_string_literal: true + +require_relative 'light/tag' + +module FicTracker::Models + class Story < Sequel::Model + # 1/week + METADATA_REFRESH_INTERVAL = 7 * 24 * 60 * 60 + # 3/day + CONTENT_REFRESH_INTERVAL = 12 * 60 * 60 + + many_to_one :author + one_to_many :chapters, order: :index + many_to_many :collection, join_table: :collection_stories + + plugin :serialization, [Light::Tag.method(:store), Light::Tag.method(:load)], :tags + plugin :serialization, :json, :data + + # Defer creation of author/chapters until story requiring them is to be saved + def before_create + if @author + @author.save unless @author.id + self.author_id = @author.id + end + + super + end + + def after_create + return if [@author, @chapters].all?(&:nil?) + + self.author = @author if @author + @author = nil + + if @chapters + latest_chapter_at = self.updated_at || self.published_at || Time.at(0) + @chapters&.each do |chapter| + latest_chapter_at = [chapter.published_at || Time.at(0), latest_chapter_at].max + add_chapter chapter + end + update(updated_at: latest_chapter_at) if latest_chapter_at > Time.at(0) && (updated_at.nil? || latest_chapter_at >= updated_at) + end + @chapters = nil + end + + def completed? + completed + end + + # Support attaching author to a not-yet-saved story + def author + @author || super + end + + def author=(author_name) + author = author_name if author_name.is_a?(FicTracker::Models::Author) + author ||= Author.find(backend_name: backend.name, slug: author_name) + author ||= backend.load_author(author_name) if id + author ||= Author.new(backend: backend, slug: author_name) + + if id + @author = nil + author.save unless author.id + super(author) + else + @author = author + end + end + + # Support attaching chapters to a not-yet-saved story + def chapters + @chapters || super + end + + def chapters=(entries) + latest_chapter_at = self.published_at || Time.at(0) + + to_add = [] + to_remove = self.chapters.map(&:id) + + entries.each do |entry| + chapter = self.chapters.find { |c| c.slug == entry[:slug] } + + if chapter + latest_chapter_at = [chapter.published_at || Time.at(0), latest_chapter_at].max + chapter.set(**entry) + else + chapter = FicTracker::Models::Chapter.new(**entry) + to_add << chapter + end + to_remove.delete chapter.id + end + + if id + @chapters = nil + to_add.each do |entry| + logger.debug "Adding new chapter #{entry.inspect} to story #{self}" + add_chapter entry + end + if to_remove.any? + logger.debug "Removing chapter(s) #{to_remove.inspect} from story #{self}" + chapter_dataset.where(id: to_remove).delete + end + + update(updated_at: latest_chapter_at) if latest_chapter_at > Time.at(0) && (updated_at.nil? || latest_chapter_at >= updated_at) + else + @chapters = (@chapters || []) + to_add - to_remove + end + end + + def ensure_fully_loaded + ensure_chapters + refresh_content + refresh_metadata + end + + def ensure_chapters + # FIXME: Should check for a reasonable set of parameters - full load unless XX% (75%?) of chapters have content + return if chapters && chapters.any? && chapters.all? { |c| c.content? && c.content_type? } + + backend.load_full_story(self) + end + + def backend + return unless backend_name + + @backend ||= FicTracker::Backends.get(backend_name) + end + + def backend=(backend) + @backend = backend + self.backend_name = backend.name + end + + def etag + chapters.select { |c| c.etag }.last + end + + def cache_key + backend.cache_key + [:s, slug] + end + + def safe_name + [backend_name, slug, name].join('_').downcase.gsub(/[^a-z0-9\-_]/, '_').gsub(/__+/, '_') + end + + def refresh_metadata + refresh_metadata! if backend && needs_metadata_refresh? + end + + def refresh_metadata! + backend.load_story(self) + end + + def refresh_content + backend.find_chapters(self) if backend && needs_content_refresh? + chapters.each(&:refresh_content) + end + + def refresh_content! + backend.find_chapters(self) + chapters.each(&:refresh_content!) + end + + def needs_metadata_refresh? + Time.now - (last_metadata_refresh || Time.at(0)) >= METADATA_REFRESH_INTERVAL + end + + def needs_content_refresh? + Time.now - (last_content_refresh || Time.at(0)) >= (completed? ? METADATA_REFRESH_INTERVAL : CONTENT_REFRESH_INTERVAL) + end + + def to_s + "#{name}, by #{author.nil? ? '' : author.to_s}" + end + + def uid + Digest::SHA1.hexdigest("#{backend}/#{slug}") + end + + private + + def logger + Logging.logger[self] + end + end +end diff --git a/lib/fic_tracker/models/user.rb b/lib/fic_tracker/models/user.rb new file mode 100644 index 0000000..07a818d --- /dev/null +++ b/lib/fic_tracker/models/user.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module FicTracker::Models + class User < Sequel::Model + plugin :serialization, :json, :data + + one_to_many :sessions + one_to_many :tracked + one_to_many :history + + end +end diff --git a/lib/fic_tracker/renderers/epub.rb b/lib/fic_tracker/renderers/epub.rb new file mode 100644 index 0000000..e1552fd --- /dev/null +++ b/lib/fic_tracker/renderers/epub.rb @@ -0,0 +1,200 @@ +# frozen_string_literal: true + +module FicTracker::Renderers + class Epub + attr_accessor :cover, :preface + attr_reader :story, :html + + def initialize(story, io:, cover: true, preface: true) + @story = story + @io = io + + @cover = cover && story.image + @preface = preface + + @html = HTML.new(story, io: nil, body_only: true) + @html.cover = 'images/cover.jpg' if @cover + end + + def render + logger.info "Rendering epub for #{story}" + + require 'zip' + require 'zip/filesystem' + + Zip::File.open_buffer(@io, create: true, compression_level: Zlib::BEST_COMPRESSION) do |zip| + zip.dir.mkdir('META-INF') + zip.file.open('META-INF/container.xml', +'w') { |f| f.write build_container_xml } + + if cover + zip.dir.mkdir('images') + zip.file.open('images/cover.jpg', +'w') { |f| f.write build_cover_image } + end + + zip.file.open('mimetype', +'w') { |f| f.write 'application/epub+zip' } + zip.file.open('content.opf', +'w') { |f| f.write build_content_opf } + zip.file.open('toc.ncx', +'w') { |f| f.write build_toc_ncx } + + zip.file.open('preface.xhtml', +'w') { |f| f.write build_preface_xhtml } if preface + + story.chapters.each do |chapter| + zip.file.open("chapter#{chapter.index}.xhtml", +'w') { |f| f.write(build_chapter_xhtml(chapter)) } + end + end + end + + private + + def logger + Logging.logger[self] + end + + def build_cover_image + require 'vips' + + buf = Net::HTTP.get story.image + im = Vips::Image.thumbnail_buffer buf, 800, height: 1280, size: :down + + im.write_to_buffer '.jpg', strip: true + end + + def build_container_xml + Nokogiri::XML::Builder.new(encoding: 'UTF-8') do |xml| + xml.container version: '1.0', xmlns: 'urn:oasis:names:tc:opendocument:xmlns:container' do + xml.rootfiles do + xml.rootfile 'full-path': 'content.opf', 'media-type': 'application/oebps-package+xml' + end + end + end.to_xml(indent: 0) + end + + def build_content_opf + Nokogiri::XML::Builder.new(encoding: 'UTF-8') do |xml| + xml.package version: '2.0', xmlns: 'http://www.idpf.org/2007/opf', 'unique-identifier': 'story-url' do + xml.metadata 'xmlns:dc': 'http://purl.org/dc/elements/1.1/', 'xmlns:opf': 'http://www.idpf.org/2007/opf' do + xml['dc'].title story.name + xml['dc'].language story.language || 'en' + xml['dc'].identifier story.url || "#{story.backend}/#{story.slug}", id: 'story-url', 'opf:scheme': 'url' + if story.synopsis + require_relative '../converters/from_html' + xml['dc'].description FicTracker::Converters::FromHTML.to_plain(@story.synopsis) + end + xml['dc'].creator story.author&.to_s || '', 'opf:role': 'aut', 'opf:file-as': (story.author&.to_s || 'Unknown') + xml['dc'].publisher story.backend.full_name + xml['dc'].date (story.published_at || Time.now).to_datetime + xml['dc'].relation story.backend.url + xml['dc'].source story.url if story.url + story.tags.each do |tag| + xml['dc'].subject tag.to_s + end + xml.meta name: 'cover', content: 'coverImage' if cover + + xml.meta name: 'ft-backend', content: story.backend_name + xml.meta name: 'ft-story', content: story.slug + xml.meta name: 'ft-etag', content: story.etag + xml.meta name: 'ft-complete', content: story.completed? + end + + xml.manifest do + xml.item id: 'coverImage', properties: 'cover-image', href: 'images/cover.jpg', 'media-type': 'image/jpeg' if cover + xml.item id: 'preface', href: 'preface.xhtml', 'media-type': 'application/xhtml+xml' if preface + story.chapters.each do |chapter| + xml.item id: "chapter#{chapter.index}", href: "chapter#{chapter.index}.xhtml", 'media-type': 'application/xhtml+xml' + end + xml.item id: 'ncx', href: 'toc.ncx', 'media-type': 'application/x-dtbncx+xml' + end + + xml.spine(toc: 'ncx') do + xml.itemref idref: 'preface', linear: 'no' if preface + story.chapters.each do |chapter| + xml.itemref idref: "chapter#{chapter.index}" + end + end + end + end.to_xml(indent: 0) + end + + def build_toc_ncx + Nokogiri::XML::Builder.new(encoding: 'UTF-8') do |xml| + xml.doc.create_internal_subset( + 'ncx', + '-//NISO//DTD ncx 2005-1//EN', + 'http://www.daisy.org/z3986/2005/ncx-2005-1.dtd' + ) + xml.ncx xmlns: 'http://www.daisy.org/z3986/2005/ncx/', version: '2005-1', 'xml:lang': story.language || 'en' do + xml.head do + xml.meta name: 'dtb:uid', content: story.url || "#{story.backend}/#{story.slug}" + xml.meta name: 'dtb:depth', content: 2 + xml.meta name: 'dtb:generator', content: "FicTracker/#{FicTracker::VERSION}" + xml.meta name: 'dtb:totalPageCount', content: 0 + xml.meta name: 'dtb:maxPageNumber', content: 0 + end + + xml.docTitle do + xml.text story.name + end + xml.docAuthor do + xml.text story.author&.to_s || '' + end + + xml.navMap do + if preface + xml.navPoint id: 'chapter0', class: 'chapter', playOrder: '0' do + xml.navLabel do + xml.text 'Preface' + end + xml.content 'preface.xhtml' + end + end + story.chapters.each do |chapter| + xml.navPoint id: "chapter#{chapter.index}", class: 'chapter', playOrder: chapter.index do + xml.navLabel do + xml.text chapter.name + end + xml.content "chapter#{chapter.index}.xhtml" + end + end + end + end + end.to_xml(indent: 0) + end + + def build_preface_xhtml + Nokogiri::XML::Builder.new(encoding: 'UTF-8') do |xml| + xml.doc.create_internal_subset( + 'html', + '-//W3C//DTD XHTML 1.1//EN', + 'http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd' + ) + xml.html xmlns: 'http://www.w3.org/1999/xhtml', 'xml:lang': story.language || 'en' do + xml.head do + xml.meta 'http-equiv': 'Content-Type', content: 'application/xhtml+xml; charset=utf-8' + xml.title story.to_s + end + xml.body do + @html.build_preface(xml) + end + end + end.to_xml(indent: 0) + end + + def build_chapter_xhtml(chapter) + Nokogiri::XML::Builder.new(encoding: 'UTF-8') do |xml| + xml.doc.create_internal_subset( + 'html', + '-//W3C//DTD XHTML 1.1//EN', + 'http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd' + ) + xml.html xmlns: 'http://www.w3.org/1999/xhtml', 'xml:lang': story.language || 'en' do + xml.head do + xml.meta 'http-equiv': 'Content-Type', content: 'application/xhtml+xml; charset=utf-8' + xml.title chapter.to_s + end + xml.body do + @html.build_chapter(xml, chapter) + end + end + end.to_xml(indent: 0) + end + end +end diff --git a/lib/fic_tracker/renderers/html.rb b/lib/fic_tracker/renderers/html.rb new file mode 100644 index 0000000..516d5bd --- /dev/null +++ b/lib/fic_tracker/renderers/html.rb @@ -0,0 +1,140 @@ +# frozen_string_literal: true + +module FicTracker::Renderers + class HTML + attr_accessor :body_only, :cover, :preface + attr_reader :story + + def initialize(story, io:, cover: false, preface: true, body_only: false) + @story = story + @io = io + + @cover = story.image if cover + @preface = preface + @body_only = body_only + end + + def render + logger.info "Rendering html for #{story}" + + doc = Nokogiri::HTML5::Document.new + Nokogiri::XML::Builder.with(doc) do |html| + build_html(html) do + html.article class: "story" do + if preface + build_preface(html) + html.hr + end + + story.chapters.each do |chapter| + html.article class: 'chapter' do + build_chapter(html, chapter) + end + html.hr unless chapter == story.chapters.last + end + end + end + end + @io.puts doc.to_html + end + + def build_preface(html) + html.article class: 'preface' do + html.h1 story.name + + html.address do + html << "By " + if story.author&.url + html.a story.author.to_s, href: story.author.url + else + html.i story.author&.to_s || '' + end + end + html.a "On #{story.backend.full_name}.", href: story.url if story.url + + html.span do + html << 'Published ' + html.time story.published_at.to_date, datetime: story.published_at.iso8601 + if story.updated_at && story.updated_at != story.published_at + html << ", #{story.completed? ? 'completed' : 'updated'} " + html.time story.updated_at.to_date, datetime: story.updated_at.iso8601 + end + html << '.' + end + html.br + + html.dl do + story.tags.map { |td| FicTracker::Models::Light::Tag.load td }.sort_by(&:ordering).group_by(&:category).each do |category, tags| + html.dt category.to_s.capitalize + html.dd do + tags.each do |tag| + if tag.important? + html.b tag.name + else + html << tag.name + end + html << ', ' unless tag == tags.last + end + end + end + end + + html.article class: 'synopsis' do + html << story.synopsis + end + + html.img alt: 'Cover image', src: cover if cover + end + end + + def build_chapter(html, chapter) + html.h2 do + html.a chapter, href: chapter.url + end + + case chapter.content_type + when 'text/html' + require_relative '../converters/from_html' + html << FicTracker::Converters::FromHTML.to_safe_html(chapter.content) + when 'text/markdown' + require_relative '../converters/from_markdown' + html << FicTracker::Converters::FromMarkdown.to_safe_html(chapter.content) + when 'text/plain' + chapter.content.split("\n\n").each do |para| + html.p para + end + else + raise "Unknown chapter content-type: #{chapter.content_type.inspect}" + end + end + + private + + def logger + Logging.logger[self] + end + + def build_html(html) + return yield if body_only + + html.html(lang: story.language || 'en') do + html.head do + html.meta charset: 'utf-8' + html.meta viewport: 'width=device-width, initial-scale=1' + html.meta name: 'author', content: story.author.to_s + html.meta name: 'generator', content: "FicTracker/#{FicTracker::VERSION}" + html.meta name: 'keywords', content: story.tags.map(&:to_s).join(',') + + html.meta name: 'ft-backend', content: story.backend_name + html.meta name: 'ft-story', content: story.slug + html.meta name: 'ft-etag', content: story.etag + + html.title story.to_s + end + html.body do + yield + end + end + end + end +end diff --git a/lib/fic_tracker/renderers/markdown.rb b/lib/fic_tracker/renderers/markdown.rb new file mode 100644 index 0000000..31d4b7b --- /dev/null +++ b/lib/fic_tracker/renderers/markdown.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +module FicTracker::Renderers + class Markdown + attr_accessor :cover, :preface + attr_reader :story + + def initialize(story, io:, cover: false, preface: true) + @story = story + @io = io + + @cover = cover && story.image + @preface = preface + end + + def render + logger.info "Rendering markdown for #{story}" + + @io.puts build_preface, nil if preface + + story.chapters.each do |chapter| + @io.puts build_chapter(chapter), nil + end + + @io.puts build_chapter_references + end + + private + + def logger + Logging.logger[self] + end + + def build_preface + require_relative '../converters/from_html' + <<~EOF + #{story.name} + #{'=' * story.name.size} + + By __*[#{story.author}][aut]*__#{' '} + Published #{story.published_at.to_date}#{(story.updated_at && story.updated_at != story.published_at) ? ", updated #{story.updated_at.to_date}" : ''}.#{' '} + [The original work.](#{story.url}) + #{cover ? "![Cover image](#{story.image})\n" : ''} + #{FicTracker::Converters::FromHTML.to_md(story.synopsis)} + EOF + end + + def build_chapter(chapter) + head = "[#{chapter}][ch#{chapter.index}]\n#{'-' * (chapter.to_s.size + 6 + chapter.index.to_s.size)}\n\n" + + head + case chapter.content_type + when 'text/html' + require_relative '../converters/from_html' + FicTracker::Converters::FromHTML.to_md(chapter.content) + when 'text/markdown' + chapter.content + when 'text/plain' + chapter.content + end + end + + def build_chapter_references + refs = [] + refs << "[aut]: #{story.author.url}" if preface + refs += story.chapters.map { |c| "[ch#{c.index}]: #{c.url}" } + refs.join "\n" + end + end +end diff --git a/lib/fic_tracker/server.rb b/lib/fic_tracker/server.rb new file mode 100644 index 0000000..2eae2d4 --- /dev/null +++ b/lib/fic_tracker/server.rb @@ -0,0 +1,162 @@ +# frozen_string_literal: true + +require 'sinatra/base' + +module FicTracker + # Web server for providing your fic tracking needs + class Server < Sinatra::Base + configure :development do + require 'sinatra/reloader' + register Sinatra::Reloader + end + + configure do + root = File.join(__dir__, '../..') + + set :views, File.join(root, 'views') + set :public_folder, File.join(root, 'public') + enable :logging + end + + helpers do + def backend + return env[:ft_backend] if env[:ft_backend] + return halt 400, 'No backend provided' unless params['backend'] + + env[:ft_backend] = Backends.get params['backend'] + return halt 400, "Unable to find backend #{params['backend']}" unless env[:ft_backend] + + env[:ft_backend] + end + end + + before do + expires 3600, :public, :must_revalidate if request.request_method == 'GET' + end + + get '/', provides: :html do + haml :index, format: :html5 + end + + get '/search/:backend', provides: :html do |_backend_name| + search = backend.get_search_info + + haml :search, format: :html5, locals: { search:, backend: } + end + + get '/search/:backend', provides: :json do |_backend_name| + backend.get_search_info.to_json + end + + post '/search/:backend', provides: :html do |_backend_name| + request.body.rewind # in case someone already read it + data = JSON.parse request.body.read, symbolize_names: true + + results = backend.sarch(data) + + haml :search_result, format: :html5, locals: { results:, backend: } + end + + post '/search/:backend', provides: :json do |_backend_name| + request.body.rewind # in case someone already read it + data = JSON.parse request.body.read, symbolize_names: true + + results = backend.sarch(data) + + results.to_json + end + + get '/author/:backend/:slug', provides: :html do |_backend_name, slug| + author = Models::Author.find(backend_name: backend.name, slug:) + author ||= Models::Author.new(backend_name: backend.name, slug:) + author.refresh_metadata + + haml :author, format: :html5, locals: { author:, backend: } + ensure + author&.save_changes + end + + get '/collection/:backend/:slug', provides: :html do |_backend_name, slug| + collection = Models::Collection.find(backend_name: backend.name, slug:) + collection ||= Models::Collection.new(backend_name: backend.name, slug:) + collection.refresh_metadata + + haml :collection, format: :html5, locals: { collection:, backend: } + ensure + collection&.save_changes + end + + get '/story/:backend/:slug', provides: :html do |_backend_name, slug| + story = Models::Story.find(backend_name: backend.name, slug:) + story ||= Models::Story.new(backend_name: backend.name, slug:) + + story.ensure_fully_loaded + story.set(last_accessed: Time.now) + + last_modified story.updated_at || story.published_at + etag story.etag + + haml :story, format: :html5, locals: { story:, backend: } + ensure + story&.save_changes + end + + get '/story/:backend/:slug/:index', provides: :html do |_backend_name, slug, index| + story = Models::Story.find(backend_name: backend.name, slug:) + story ||= Models::Story.new(backend_name: backend.name, slug:) + + story.refresh_content + story.refresh_metadata + story.set(last_accessed: Time.now) + + chapter = story.chapters[index.to_i] + + last_modified chapter.published_at + etag chapter.etag + + haml :chapter, format: :html5, locals: { chapter:, story:, backend: } + ensure + story&.save_changes + end + + # rubocop:disable Metrics/BlockLength + get '/story/:backend/:slug.:format' do |_backend_name, slug, format| + render_klass = nil + mime = nil + case format + when 'epub', :epub + render_klass = Renderers::Epub + mime = 'application/epub+zip' + when 'html', :html + render_klass = Renderers::HTML + mime = 'text/html' + when 'txt', :txt, 'md', :md + render_klass = Renderers::Markdown + mime = 'text/markdown' + else + halt 400, "Unknown format #{format}" + end + + story = Models::Story.find(backend_name: backend.name, slug:) + story ||= Models::Story.new(backend_name: backend.name, slug:) + + story.refresh_content + story.refresh_metadata + story.set(last_accessed: Time.now) + + attachment "#{story.safe_name}.#{format}" + content_type mime + + last_modified story.updated_at || story.published_at + etag story.etag + + story.ensure_fully_loaded + stream do |out| + render_klass.new(story, io: out).render + end + ensure + story&.save_changes + end + # rubocop:enable Metrics/BlockLength + end +end diff --git a/lib/fic_tracker/tasks/cleanup.rb b/lib/fic_tracker/tasks/cleanup.rb new file mode 100644 index 0000000..0948683 --- /dev/null +++ b/lib/fic_tracker/tasks/cleanup.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module FicTracker::Tasks + module Cleanup + # Remove stories that haven't been accessed in a month + DELETE_UNACCESSED_STORIES = 30 * 24 * 60 * 60 + + def self.run + delete_unaccessed_stories + delete_unnecessary_authors + FicTracker.cache.expire + end + + class <1.5KB unless otherwise specified + @compress = compress || 1500 + @compress = (@compress..) if @compress.is_a? Numeric + @encoding = encoding + end + + # Expire any unwanted entries from the cache + def expire + @backend.expire if @backend.respond_to? :expire + end + + # Check if the cache contains a given key which has not expired + def has?(key) + key = expand_key(key) + return @backend.has?(key) if @backend.respond_to? :has? + + !@backend.get(key).nil? + end + # Get the value of a key, will return nil if the key is expired + def get(key) + key = expand_key(key) + decode_data @backend.get(key) + end + # Set the value for a given key, with an optional expiry + def set(key, value, expire = nil) + key = expand_key(key) + @backend.set(key, encode_data(value), expire) + end + # Get the value for a given key, or set it as the result of the block with an optional expiry + def get_or_set(key, expire = nil, &block) + key = expand_key(key) + return decode_data(@backend.get_or_set(key, expire) { encode_data(block.call) }) if @backend.respond_to? :get_or_set + return get(key) if has?(key) + + value = block.call + + @backend.set(key, encode_data(value), expire) + + value + end + # Clear the value for a given key + def delete(key) + key = expand_key(key) + @backend.delete key if @backend.respond_to? :delete + end + # Clear the entire cache + def clear + @backend.clear if @backend.respond_to? :clear + end + + private + + def logger + Logging.logger[self] + end + + attr_accessor :encoder, :compress + + ENCODING_FLAGS = 1 << 0 | 1 << 1 | 1 << 2 + COMPRESSED_FLAG = 1 << 3 + + ENCODING_NONE = 0 + ENCODING_MARSHAL = 1 + ENCODING_JSON = 2 + ENCODING_CBOR = 3 + ENCODING_CUSTOM = 4 + + def expand_key(key) + return key.join '/' if key.is_a? Array + + key + end + + # Check if a piece of data is a plain-old-object - i.e. simple data + def is_pod?(data) + return true if data.nil? || data == true || data == false || data.is_a?(String) || data.is_a?(Numeric) + return true if data.is_a?(Array) && data.all? { |k| is_pod?(k) } + return true if data.is_a?(Hash && data.all? { |k, v| k.is_a?(Symbol) && is_pod?(v) }) + + false + end + + def best_encoder?(data) + pod_encoder = :none if data.is_a?(String) + pod_encoder ||= :cbor if Object.const_defined?(:CBOR) + pod_encoder ||= :json if Object.const_defined?(:JSON) + pod_encoder ||= :marshal + return pod_encoder if is_pod?(data) + + nil + end + + def encode_data(data, encode: nil) + encode ||= @encoder + encode ||= best_encoder?(data) + + flags = 0 + case encode + when :none + data = data + flags |= ENCODING_NONE + when :marshal + data = Marshal.dump(data) + flags |= ENCODING_MARSHAL + when :json + data = data.to_json + flags |= ENCODING_JSON + when :cbor + data = data.to_cbor + flags |= ENCODING_CBOR + else + raise "Unknown encoder #{(encode || @encoder).inspect}" unless encode.nil? && @encoder.respond_to?(:dump) + + @encode.dump(data) + flags |= ENCODING_CUSTOM + end + + if @compress && @compress.include?(data.bytesize) + require 'zlib' + data = Zlib::Deflate.deflate(data, Zlib::BEST_COMPRESSION) + flags |= COMPRESSED_FLAG + end + + data.dup.prepend([flags].pack('C')) + end + + def decode_data(data) + flags = data[0].unpack('C').first + data = data[1..] + + if (flags & COMPRESSED_FLAG) == COMPRESSED_FLAG + require 'zlib' + data = Zlib::Inflate.inflate(data) + data.force_encoding('UTF-8') + end + + case (flags & ENCODING_FLAGS) + when ENCODING_MARSHAL + Marshal.load(data) + when ENCODING_JSON + JSON.parse(data, symbolize_names: true) + when ENCODING_CBOR + CBOR.decode(data) + when ENCODING_CUSTOM + @encode.load(data) + else + data + end + end + end +end diff --git a/lib/fic_tracker/util/cache/base.rb b/lib/fic_tracker/util/cache/base.rb new file mode 100644 index 0000000..2d382ba --- /dev/null +++ b/lib/fic_tracker/util/cache/base.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module FicTracker::Util::CacheImpl + class Base + def has?(key); end + def get(key); end + def set(key, value, ttl); end + def delete(key); end + end +end + diff --git a/lib/fic_tracker/util/cache/database.rb b/lib/fic_tracker/util/cache/database.rb new file mode 100644 index 0000000..1a2ca1e --- /dev/null +++ b/lib/fic_tracker/util/cache/database.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +module FicTracker::Util::CacheImpl + # A cache using live redis data + class Database < Base + attr_reader :dataset, :dataset_expired, :dataset_live + + def initialize(table: 'cache', **redis) + @dataset = FicTracker.database[table.to_s.to_sym] + @dataset_expired = @dataset.exclude{ Sequel.|({ expire_at: nil }, Sequel::CURRENT_DATE < expire_at) } + @dataset_live = @dataset.where{ Sequel.|({ expire_at: nil }, Sequel::CURRENT_DATE < expire_at) } + end + + def expire + dataset_expired.delete + end + + def has?(key) + dataset_live.filter(key: key).any? + end + + def get(key) + blob = dataset_live.filter(key: key).first&.[](:value) + return unless blob + + blob.to_s + end + + def set(key, value, expire = nil) + expire = expire >= 0 ? Time.now + expire : nil if expire.is_a?(Numeric) + expire = nil if expire.is_a?(Numeric) + + blob = Sequel.blob(value) + dataset + .insert_conflict(target: :key, update: { value: blob, expire_at: expire }) + .insert(key: key, value: blob, expire_at: expire) + + value + end + + def delete(key) + dataset.where(key: key).delete + nil + end + + def clear + dataset.delete + end + + private + + def namespace_key(key) + "#{namespace}-#{key}" + end + + def unnamespace_key(key) + key.delete_prefix("#{@namespace}-") + end + end +end diff --git a/lib/fic_tracker/util/cache/dummy.rb b/lib/fic_tracker/util/cache/dummy.rb new file mode 100644 index 0000000..3d2a83d --- /dev/null +++ b/lib/fic_tracker/util/cache/dummy.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +module FicTracker::Util::CacheImpl + # A dummy cache that doesn't store any data + class Dummy < Base + end +end diff --git a/lib/fic_tracker/util/cache/file.rb b/lib/fic_tracker/util/cache/file.rb new file mode 100644 index 0000000..ea13ed8 --- /dev/null +++ b/lib/fic_tracker/util/cache/file.rb @@ -0,0 +1,99 @@ +# frozen_string_literal: true + +require 'fileutils' +require 'tmpdir' + +module FicTracker::Util::CacheImpl + # A filesystem-backed in-memory cache + class File < Base + def initialize(dir: nil) + @dir = dir || Dir.mktmpdir('ficagg') + + @internal = Memory.new + end + + def expire + @internal.lock do + Dir[File.join(@dir, '**')].each_child do |entry| + next unless entry.end_with?('.meta') + + meta = JSON.parse(File.read(entry), symbolize_names: true) + unset meta[:key] unless meta_valid?(meta) + end + end + end + + def has?(key) + internal_has = @internal.has?(key) + return internal_has if internal_has + + File.exist?(key_path(key)) && key_valid?(key) + end + + def get(key) + # TODO: Extract the metadata ttl into the memory cache + @internal.get_or_set(key, expiry: Time.at(0)) do + File.read(key_path(key)) if has?(key) + end + end + + def set(key, value, expiry = nil) + expiry = expiry >= 0 ? Time.now + expiry : nil if expiry.is_a?(Numeric) + expiry = nil if expiry.is_a?(Numeric) + meta = { + key: key, + ttl: expiry&.to_i + }.compact.to_json + + @internal.unset(key) + @internal.get_or_set(key, expiry: expiry) do + File.write(key_path(key), value) + File.write(meta_path(key), meta) + value + end + end + + def delete(key) + @internal.unset(key) + @internal.lock do + [key_path(key), meta_path(key)].each do |path| + File.delete(path) if File.exist?(path) + end + end + end + + def clear + @internal.clear + @internal.lock do + Dir[File.join(@dir, '**')].each_child { |entry| File.delete(File.join(@dir, entry)) } + end + end + + private + + def key_path(key) + File.join(@dir, safe_key(key)) + end + def meta_path(key) + File.join(@dir, "#{safe_key(key)}.meta") + end + + def key_valid?(key) + path = meta_path(key) + return unless File.exist?(path) + + meta = JSON.parse(File.read(path), symbolize_names: true) + meta_valid?(meta) + end + + def meta_valid?(meta) + return true unless meta[:ttl] + + Time.now < Time.at(meta[:ttl]) + end + + def safe_key(key) + key.gsub(%r{[^A-Za-z0-9-]}, '_') + end + end +end diff --git a/lib/fic_tracker/util/cache/memory.rb b/lib/fic_tracker/util/cache/memory.rb new file mode 100644 index 0000000..9cb3459 --- /dev/null +++ b/lib/fic_tracker/util/cache/memory.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +module FicTracker::Util::CacheImpl + # A threadsafe in-memory cache + class Memory < Base + Entry = Struct.new('Entry', :data, :ttl) do + def valid? + return true unless ttl + + Time.now < ttl + end + + def expired? + return false unless ttl + + Time.now >= ttl + end + end + + def initialize + @data = {} + @mutex = Thread::Mutex.new + end + + def lock(&block) + @mutex.synchronize { block.call } + end + + def expire + @mutex.synchronize do + @data.delete_if { |_, v| v.expired? } + end + nil + end + + def has?(key) + @mutex.synchronize do + @data[key]&.valid? + end + end + + def get(key) + @mutex.synchronize do + @data[key].data if has?(key) + end + end + + def set(key, value, expiry = nil) + @mutex.synchronize do + expiry = expiry >= 0 ? Time.now + expiry : nil if expiry.is_a?(Numeric) + expiry = nil if expiry.is_a?(Numeric) + + @data[key] = Entry.new(value, expiry) + value + end + end + + def get_or_set(key, expiry: -1, &block) + @mutex.synchronize do + return @data[key].data if has?(key) + + value = yield + expiry = expiry >= 0 ? Time.now + expiry : nil if expiry.is_a?(Numeric) + expiry = nil if expiry.is_a?(Numeric) + + @data[key] = Entry.new(value, expiry) + value + end + end + + def delete(key) + @mutex.synchronize do + @data.delete key + end + nil + end + + def clear + @mutex.synchronize do + @data.clear + end + nil + end + end +end diff --git a/lib/fic_tracker/util/cache/redis.rb b/lib/fic_tracker/util/cache/redis.rb new file mode 100644 index 0000000..2d4f5ae --- /dev/null +++ b/lib/fic_tracker/util/cache/redis.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +require 'redis' + +module FicTracker::Util::CacheImpl + class Redis < Base + # A cache using live redis data + def initialize(namespace: 'cache', **redis) + @client = Redis.new(**redis) + + @namespace = namespace + end + + def has?(key) + @client.exists?(namespace_key(key)) + end + + def get(key) + @client.get(namespace_key(key)) + end + + def set(key, value, expiry = nil) + key = namespace_key(key) + + expiry = expiry >= 0 ? Time.now + expiry : nil if expiry.is_a?(Numeric) + expiry = nil if expiry.is_a?(Numeric) + + @client.set(key, value, exat: expiry) + + value + end + + def delete(key) + key = namespace_key(key) + @client.del key + nil + end + + def clear + @client.scan_each(match: "#{@namespace}-*") { |key| @client.del key } + end + + def each + @data.scan_each(match: "#{@namespace}-*") { |key| yield @client.get(key) } + end + + private + + def namespace_key(key) + "#{namespace}-#{key}" + end + + def unnamespace_key(key) + key.delete_prefix("#{@namespace}-") + end + end +end diff --git a/lib/fic_tracker/util/database.rb b/lib/fic_tracker/util/database.rb new file mode 100644 index 0000000..ec9936a --- /dev/null +++ b/lib/fic_tracker/util/database.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +require 'json' +require 'sequel' +require 'sequel/plugins/serialization' + +module FicTracker::Util + class Database + DEFAULT_SESSION_TIMEOUT = 48 * 60 * 60 + + def self.connect(connection_string, migrate: true, options: {}) + raise 'Must provide a database URL' if connection_string.nil? || connection_string.empty? + + db = if connection_string == :memory + Sequel.sqlite + else + Sequel.connect(connection_string, **options) + end + + db.sql_log_level = :debug + + # Defer logging until after migrations for in-memory store + db.loggers << Logging.logger[self] unless connection_string == :memory + self.migrate(db) if migrate + db.loggers << Logging.logger[self] if connection_string == :memory + + db + end + + def self.migrate(db) + Sequel.extension :migration + Sequel::Migrator.run(db, File.join(__dir__, '..', 'migrations')) + end + end +end diff --git a/lib/fic_tracker/util/hash_extensions.rb b/lib/fic_tracker/util/hash_extensions.rb new file mode 100644 index 0000000..6e68bcc --- /dev/null +++ b/lib/fic_tracker/util/hash_extensions.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +class Hash + def deep_transform_keys(&block) + to_h { |k, v| [block.call(k), v.is_a?(Hash) ? v.deep_transform_keys(&block) : v] } + end +end diff --git a/lib/fic_tracker/version.rb b/lib/fic_tracker/version.rb new file mode 100644 index 0000000..76bc538 --- /dev/null +++ b/lib/fic_tracker/version.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +module FicTracker + VERSION = "0.1.0" +end diff --git a/test/test_fic_tracker.rb b/test/test_fic_tracker.rb new file mode 100644 index 0000000..bdcbc99 --- /dev/null +++ b/test/test_fic_tracker.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +require "test_helper" + +class TestFicTracker < Minitest::Test + def test_that_it_has_a_version_number + refute_nil ::FicTracker::VERSION + end + + def test_it_does_something_useful + assert false + end +end diff --git a/test/test_helper.rb b/test/test_helper.rb new file mode 100644 index 0000000..88dee09 --- /dev/null +++ b/test/test_helper.rb @@ -0,0 +1,6 @@ +# frozen_string_literal: true + +$LOAD_PATH.unshift File.expand_path("../lib", __dir__) +require "fic_tracker" + +require "minitest/autorun" diff --git a/views/author.haml b/views/author.haml new file mode 100644 index 0000000..e69de29 diff --git a/views/chapter.haml b/views/chapter.haml new file mode 100644 index 0000000..e69de29 diff --git a/views/index.haml b/views/index.haml new file mode 100644 index 0000000..e69de29 diff --git a/views/story.haml b/views/story.haml new file mode 100644 index 0000000..e69de29