diff --git a/bin/fic_tracker b/bin/fic_tracker index 4bf9493..c739d4c 100755 --- a/bin/fic_tracker +++ b/bin/fic_tracker @@ -6,9 +6,9 @@ require 'optparse' require 'ostruct' options = OpenStruct.new -OptParse.new do |opts| +optparse = OptParse.new do |opts| opts.banner = 'Usage: fic_tracker [OPTIONS...]' - + opts.on '-b', '--backend=BACKEND', 'The backend to use' do |backend| options.backend = backend end @@ -56,13 +56,24 @@ OptParse.new do |opts| puts FicTracker::VERSION exit end -end.parse! +end +optparse.parse! + +unless options.backend && options.format && options.story + puts "Backend, format, and sotry must be provided\n" + puts optparse + exit 1 +end FicTracker.logger.level = options.log_level || :warn FicTracker.configure! backend = FicTracker::Backends.get(options.backend) -story = FicTracker::Models::Story.find(backend_name: backend.name, slug: backend.parse_slug(options.story)) || FicTracker::Models::Story.new(backend: backend, slug: backend.parse_slug(options.story)) + +slug = options.story +slug = backend.parse_slug(slug) if backend.respond_to? :parse_slug + +story = FicTracker::Models::Story.find(backend_name: backend.name, slug:) || FicTracker::Models::Story.new(backend:, slug:) story.ensure_fully_loaded data = nil diff --git a/lib/fic_tracker.rb b/lib/fic_tracker.rb index cd76569..f1378ac 100644 --- a/lib/fic_tracker.rb +++ b/lib/fic_tracker.rb @@ -51,52 +51,9 @@ module FicTracker @global_logger ||= false end - module Models - def self.const_missing(const) - raise 'No database connected' unless FicTracker.database - - model = const.to_s.downcase - require_relative "fic_tracker/models/#{model}" - - mod = const_get(const) if const_defined? const - return mod if mod - - raise "Model not found: #{const}" - end - end - - module Tasks - autoload :Cleanup, 'fic_tracker/tasks/cleanup' - end - - module Renderers - autoload :Epub, 'fic_tracker/renderers/epub' - autoload :HTML, 'fic_tracker/renderers/html' - autoload :Markdown, 'fic_tracker/renderers/markdown' - - def self.render(type, story, **attrs) - klass = case type - when :Markdown, :markdown, :md - Markdown - when :HTML, :html - HTML - when :Epub, :epub - Epub - end - - stringio = nil - unless attrs[:io] - require 'stringio' - attrs[:io] = StringIO.new - stringio = true - end - - result = klass.new(story, **attrs).render - return result unless stringio - - attrs[:io].string - end - end + autoload :Models, 'fic_tracker/models' + autoload :Renderers, 'fic_tracker/renderers' + autoload :Tasks, 'fic_tracker/tasks' end require_relative 'fic_tracker/backend' diff --git a/lib/fic_tracker/backends/ao3/backend.rb b/lib/fic_tracker/backends/ao3/backend.rb index a344e8c..f087d33 100644 --- a/lib/fic_tracker/backends/ao3/backend.rb +++ b/lib/fic_tracker/backends/ao3/backend.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require_relative 'client' +require_relative 'search_info' module FicTracker::Backends::Ao3 class Backend < FicTracker::Backend @@ -9,7 +10,7 @@ module FicTracker::Backends::Ao3 category: -2, warning: -1, }.freeze - + def client @client ||= Client.new end @@ -53,7 +54,7 @@ module FicTracker::Backends::Ao3 story = FicTracker::Models::Story.new(slug: parse_slug(story), backend: self) unless story.is_a? FicTracker::Models::Story logger.info "Loading story #{story.slug}" - doc = client.request("/works/#{story.slug}") + doc = client.request("/works/#{story.slug}", query: { view_adult: true }) attrs = extract_story doc @@ -96,7 +97,7 @@ module FicTracker::Backends::Ao3 story = FicTracker::Models::Story.new(slug: parse_slug(story), backend: self) unless story.is_a? FicTracker::Models::Story logger.info "Loading all chapters for #{story.slug}" - doc = client.request("/works/#{story.slug}", query: { view_full_work: true }) + doc = client.request("/works/#{story.slug}", query: { view_full_work: true, view_adult: true }) attrs = extract_story(doc) chapters = doc.css('#chapters > div.chapter').map { |chapter| extract_chapter(chapter) } @@ -116,13 +117,23 @@ module FicTracker::Backends::Ao3 chapter = FicTracker::Models::Chapter.new(slug: parse_slug(chapter), story: story) unless chapter.is_a? FicTracker::Models::Chapter logger.info "Loading chapter #{chapter.slug} for #{story.slug}" - doc = client.request("/works/#{story.slug}/chapters/#{chapter.slug}") + doc = client.request("/works/#{story.slug}/chapters/#{chapter.slug}", query: { view_adult: true }) attrs = extract_chapter(doc.at_css('#chapters > div.chapter')) chapter.set(**attrs) end + def get_search_info + SearchInfo.new(self) + end + + def search(search_info) + info = SearchInfo.from_search(search_info) + + info + end + def parse_slug(slug) return URI(slug).path.split('/').last if slug.is_a?(String) && slug.start_with?('http') @@ -176,15 +187,15 @@ module FicTracker::Backends::Ao3 published_at = Time.parse(published_at) if published_at updated_at = meta.at_css('dl.stats dd.status')&.text&.strip updated_at = Time.parse(updated_at) if updated_at - + { name: name, authors: authors, synopsis: synopsis, url: url.to_s, language: language, - chapter_count: chapters.first.to_i, - word_count: words, + chapter_count: chapters.first.to_i, + word_count: words, completed: chapters.first == chapters.last, published_at: published_at, updated_at: updated_at, @@ -208,7 +219,7 @@ module FicTracker::Backends::Ao3 title = title_extra.empty? ? title_base : title_extra end - { + { index: index, slug: slug, diff --git a/lib/fic_tracker/backends/ao3/search_info.rb b/lib/fic_tracker/backends/ao3/search_info.rb new file mode 100644 index 0000000..8c6a725 --- /dev/null +++ b/lib/fic_tracker/backends/ao3/search_info.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +require 'fic_tracker/models/light/search_info' + +module FicTracker::Backends::Ao3 + class SearchInfo < FicTracker::Models::Light::SearchInfo + # Meta tags + tag_category :category, list: [ + {name: 'Gen'}, + {} + ] + tag_category :rating, list: [ + {}, + {} + ] + + tag_category :character, freeform: true + tag_category :fandom, search: ->(search) { autocomplete_fandom(search) } + tag_category :relationship, freeform: true + tag_category :freeform, freeform: true + + def initialize(backend) + @backend = backend + @client = backend.client + end + + private + + def autocomplete_fandom(name) + @client.request("/autocomplete").map do |tag| + end + end + end +end diff --git a/lib/fic_tracker/models.rb b/lib/fic_tracker/models.rb new file mode 100644 index 0000000..f03bd5e --- /dev/null +++ b/lib/fic_tracker/models.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module FicTracker::Models + def self.const_missing(const) + raise 'No database connected' unless FicTracker.database + + model = const.to_s.downcase + require_relative "models/#{model}" + + mod = const_get(const) if const_defined? const + return mod if mod + + raise "Model not found: #{const}" + end +end diff --git a/lib/fic_tracker/models/light/search_info.rb b/lib/fic_tracker/models/light/search_info.rb index 78be963..929b2eb 100644 --- a/lib/fic_tracker/models/light/search_info.rb +++ b/lib/fic_tracker/models/light/search_info.rb @@ -1,8 +1,17 @@ # frozen_string_literal: true - module FicTracker::Models::Light class SearchInfo + def initialize(**data) + data.each do |k, v| + send(:"#{k}=", v) if respond_to?(:"#{k}=") + end + end + + def with_language(lang) + self + end + def with_tag(tag) self end @@ -15,10 +24,56 @@ module FicTracker::Models::Light self end + def with_cursor(cursor) + self + end + + def find_tags(category, search = nil) + raise ArgumentError, "No such category #{category.inspect}" unless @tags.key? category + + tag_cat = @tags[category] + return if tag_cat[:freeform] + + list = begin + if tag_cat[:list] + return tag_cat[:list] if search.nil? + return tag_cat[:list].select do |t| + case search + when Regex + t[:name] =~ search + when String + t[:name].downcase.include? search.downcase + else + false + end + end + end + return tag_cat[:search].call(search) if tag_cat[:search] + end + list.each do |t| + t[:id] ||= t[:name] + t[:category] ||= category + end + FicTracker::Models::Light::Tag.load list + end + + def to_info_json + self.class.to_info_json + end + class << self + def from_search(data) + + end + + def to_info_json + + end + protected - + def tag_category(category, **params) + (@tags ||= {})[category] = params end def word_limits(*limits) diff --git a/lib/fic_tracker/models/story.rb b/lib/fic_tracker/models/story.rb index 9c07136..95c10f0 100644 --- a/lib/fic_tracker/models/story.rb +++ b/lib/fic_tracker/models/story.rb @@ -64,7 +64,7 @@ module FicTracker::Models author ||= Author.find(backend_name: backend.name, slug: author_name) author ||= backend.load_author(author_name) if id author ||= Author.new(backend: backend, slug: author_name) - + self.authors = [author] end @@ -79,7 +79,7 @@ module FicTracker::Models to_add << author else author = self.authors.find { |c| c.slug == entry[:slug] } - + if author author.set(**entry) else @@ -110,7 +110,7 @@ module FicTracker::Models def chapters @chapters || super end - + def chapters=(entries) latest_chapter_at = self.published_at || Time.at(0) @@ -119,7 +119,7 @@ module FicTracker::Models entries.each do |entry| chapter = self.chapters.find { |c| c.slug == entry[:slug] } - + if chapter latest_chapter_at = [chapter.published_at || Time.at(0), latest_chapter_at].max chapter.set(**entry) @@ -148,16 +148,17 @@ module FicTracker::Models end def ensure_fully_loaded - ensure_chapters - refresh_content - refresh_metadata + # FIXME: Should check for a reasonable set of parameters - full load unless XX% (75%?) of chapters have content + if chapters && chapters.any? && chapters.all? { |c| c.content? && c.content_type? } + refresh_content + refresh_metadata + else + backend.load_full_story(self) + end end def ensure_chapters - # FIXME: Should check for a reasonable set of parameters - full load unless XX% (75%?) of chapters have content - return if chapters && chapters.any? && chapters.all? { |c| c.content? && c.content_type? } - - backend.load_full_story(self) + ensure_fully_loaded end def backend diff --git a/lib/fic_tracker/renderers.rb b/lib/fic_tracker/renderers.rb new file mode 100644 index 0000000..0fc7590 --- /dev/null +++ b/lib/fic_tracker/renderers.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module FicTracker::Renderers + autoload :Epub, 'fic_tracker/renderers/epub' + autoload :HTML, 'fic_tracker/renderers/html' + autoload :Markdown, 'fic_tracker/renderers/markdown' + + def self.render(type, story, **attrs) + klass = case type + when :Markdown, :markdown, :md + Markdown + when :HTML, :html + HTML + when :Epub, :epub + Epub + end + + stringio = nil + unless attrs[:io] + require 'stringio' + attrs[:io] = StringIO.new + stringio = true + end + + result = klass.new(story, **attrs).render + return result unless stringio + + attrs[:io].string + end +end diff --git a/lib/fic_tracker/renderers/epub.rb b/lib/fic_tracker/renderers/epub.rb index 5654cd0..e2357ed 100644 --- a/lib/fic_tracker/renderers/epub.rb +++ b/lib/fic_tracker/renderers/epub.rb @@ -69,6 +69,8 @@ module FicTracker::Renderers end def build_content_opf + require_relative '../util/time_extensions' + Nokogiri::XML::Builder.new(encoding: 'UTF-8') do |xml| xml.package version: '2.0', xmlns: 'http://www.idpf.org/2007/opf', 'unique-identifier': 'story-url' do xml.metadata 'xmlns:dc': 'http://purl.org/dc/elements/1.1/', 'xmlns:opf': 'http://www.idpf.org/2007/opf' do @@ -95,6 +97,7 @@ module FicTracker::Renderers xml.meta name: 'ft-backend', content: story.backend_name xml.meta name: 'ft-story', content: story.slug xml.meta name: 'ft-etag', content: story.etag + xml.meta name: 'ft-modified', content: (story.updated_at || story.published_at || Time.now).to_header xml.meta name: 'ft-complete', content: story.completed? end @@ -120,7 +123,7 @@ module FicTracker::Renderers def build_toc_ncx Nokogiri::XML::Builder.new(encoding: 'UTF-8') do |xml| xml.doc.create_internal_subset( - 'ncx', + 'ncx', '-//NISO//DTD ncx 2005-1//EN', 'http://www.daisy.org/z3986/2005/ncx-2005-1.dtd' ) diff --git a/lib/fic_tracker/renderers/html.rb b/lib/fic_tracker/renderers/html.rb index be90987..6ed0a70 100644 --- a/lib/fic_tracker/renderers/html.rb +++ b/lib/fic_tracker/renderers/html.rb @@ -112,7 +112,7 @@ module FicTracker::Renderers raise "Unknown chapter content-type: #{chapter.content_type.inspect}" end end - + private def logger @@ -122,10 +122,12 @@ module FicTracker::Renderers def build_html(html) return yield if body_only + require_relative '../util/time_extensions' + html.html(lang: story.language || 'en') do html.head do html.meta charset: 'utf-8' - html.meta viewport: 'width=device-width, initial-scale=1' + html.meta name: 'viewport', content: 'width=device-width, initial-scale=1' story.authors.each do |aut| html.meta name: 'author', content: aut.to_s end @@ -135,6 +137,8 @@ module FicTracker::Renderers html.meta name: 'ft-backend', content: story.backend_name html.meta name: 'ft-story', content: story.slug html.meta name: 'ft-etag', content: story.etag + html.meta name: 'ft-modified', content: (story.updated_at || story.published_at || Time.now).to_header + html.meta name: 'ft-complete', content: story.completed? html.title story.to_s end diff --git a/lib/fic_tracker/server.rb b/lib/fic_tracker/server.rb index d2c7129..0d13341 100644 --- a/lib/fic_tracker/server.rb +++ b/lib/fic_tracker/server.rb @@ -51,7 +51,13 @@ module FicTracker end get '/search/:backend', provides: :json do |_backend_name| - backend.get_search_info.to_json + search = backend.get_search_info + if params['tag'] || params['name'] + raise "Missing params, must specify both tag and name" unless params['tag'] && params['name'] + return search.class.find_tags(params['tag'], params['name']).to_json + end + + search.to_info_json end post '/search/:backend', provides: :html do |_backend_name| @@ -108,14 +114,16 @@ module FicTracker halt 400, "Unknown format #{format}" end - story = Models::Story.find(backend_name: backend.name, slug:) - story ||= Models::Story.new(backend_name: backend.name, slug:) - content_type mime attachment "#{story.safe_name}.#{format}" - last_modified story.updated_at || story.published_at - etag story.etag + story = Models::Story.find(backend_name: backend.name, slug:) + if story + story.set(last_accessed: Time.now) + + last_modified story.updated_at || story.published_at + etag story.etag + end ensure story&.save_changes end @@ -140,8 +148,7 @@ module FicTracker story = Models::Story.find(backend_name: backend.name, slug:) story ||= Models::Story.new(backend_name: backend.name, slug:) - story.refresh_content - story.refresh_metadata + story.ensure_fully_loaded unless story.chapters&.any? story.set(last_accessed: Time.now) content_type mime @@ -176,8 +183,7 @@ module FicTracker story = Models::Story.find(backend_name: backend.name, slug:) story ||= Models::Story.new(backend_name: backend.name, slug:) - story.refresh_content - story.refresh_metadata + story.ensure_fully_loaded story.set(last_accessed: Time.now) chapter = story.chapters[index.to_i] diff --git a/lib/fic_tracker/tasks.rb b/lib/fic_tracker/tasks.rb new file mode 100644 index 0000000..4ade16e --- /dev/null +++ b/lib/fic_tracker/tasks.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +module FicTracker::Tasks + autoload :Cleanup, 'fic_tracker/tasks/cleanup' +end diff --git a/lib/fic_tracker/util/time_extensions.rb b/lib/fic_tracker/util/time_extensions.rb new file mode 100644 index 0000000..d3d78ec --- /dev/null +++ b/lib/fic_tracker/util/time_extensions.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +class Time + def to_header + getgm.strftime("%a, %d %b %Y %T GMT") + end +end