Fix fetching original illust images

main.cpp

@@ -72,6 +72,26 @@ int main(int argc, char** argv) {
         serve_redirect(req, res, config, get_origin(req, config) + "/tags/" + blankie::murl::escape(std::move(q)));
     });
 
+    server.Get("/guess_extension/i.pximg.net(/.+)", [&](const httplib::Request& req, httplib::Response& res) {
+        using namespace std::string_literals;
+
+        // rip query parameters, but they're not important anyway
+        std::string path = req.matches.str(1);
+        auto serve_extension = [&](const char* extension) {
+            if (!pixiv_client.i_pximg_url_valid(path + '.' + extension)) {
+                return false;
+            }
+            serve_redirect(req, res, config, proxy_image_url(req, config, "https://i.pximg.net" + path + '.' + extension), true);
+            return true;
+        };
+
+        if (serve_extension("png") || serve_extension("jpg")) {
+            return;
+        }
+        res.status = 500;
+        serve_error(req, res, config, "500: Internal Server Error", "Failed to guess file extension for https://i.pximg.net"s + path);
+    });
+
 #ifndef NDEBUG
     server.Get("/debug/exception/known", [](const httplib::Request& req, httplib::Response& res) {
         throw std::runtime_error("awoo");

pixivclient.cpp

@@ -10,6 +10,10 @@ PixivClient::PixivClient() {
     this->_www_pixiv_net_client.set_default_headers({
         {"Cookie", "webp_available=1"}
     });
+    this->_i_pximg_net_client.set_keep_alive(true);
+    this->_i_pximg_net_client.set_default_headers({
+        {"Referer", "https://www.pixiv.net/"}
+    });
 }
 
 User PixivClient::get_user(uint64_t user_id) {
@@ -83,3 +87,11 @@ nlohmann::json PixivClient::_handle_result(httplib::Result res) {
     }
     return j.at("body");
 }
+
+bool PixivClient::i_pximg_url_valid(const std::string& path) {
+    httplib::Result res = this->_i_pximg_net_client.Head(path, {{"User-Agent", touch_user_agent}});
+    if (!res) {
+        throw HTTPLibException(res.error());
+    }
+    return res->status >= 200 && res->status <= 200;
+}

pixivclient.h

@@ -17,9 +17,12 @@ public:
     SearchResults search_illusts(const std::string& query, size_t page, const std::string& order);
     std::vector<SearchSuggestion> get_search_suggestions(const std::string& query);
 
+    bool i_pximg_url_valid(const std::string& path);
+
 private:
     nlohmann::json _handle_result(httplib::Result res);
     httplib::Client _www_pixiv_net_client{"https://www.pixiv.net"};
+    httplib::Client _i_pximg_net_client{"https://i.pximg.net"};
 };
 
 class HTTPLibException : public std::exception {

pixivmodels.cpp

@@ -9,6 +9,7 @@ static inline std::optional<std::string> get_1920x960_cover_image(blankie::murl:
 static inline std::optional<std::string> get_original_cover_image(blankie::murl::Url url, const nlohmann::json& cover_image);
 static inline std::optional<std::string> get_original_profile_picture(blankie::murl::Url url);
 static inline std::optional<std::string> get_360x360_illust_thumbnail(blankie::murl::Url url);
+static inline std::optional<std::string> get_original_illust_image(blankie::murl::Url url);
 static Images get_profile_pictures(const nlohmann::json& j);
 static Images get_profile_pictures(const std::string& url);
 static std::optional<std::pair<uint64_t, uint64_t>> get_thumbnail_size(blankie::murl::Url thumbnail_url, std::optional<std::pair<uint64_t, uint64_t>> original_size);
@@ -286,6 +287,19 @@ static inline std::optional<std::string> get_360x360_illust_thumbnail(blankie::m
     return url.to_string();
 }
 
+static std::regex illust_original_image_path_regex("/c/.+?/(img/.+)(?:_master1200|square1200)\\.\\w{3,4}");
+static inline std::optional<std::string> get_original_illust_image(blankie::murl::Url url) {
+    using namespace std::string_literals;
+
+    std::smatch sm;
+    if (!std::regex_match(url.path, sm, illust_original_image_path_regex)) {
+        return std::nullopt;
+    }
+    url.path = "/img-original/"s + sm.str(1);
+    url.fragment = "guess_extension";
+    return url.to_string();
+}
+
 static std::regex illust_size_regex(
     "/c/(\\d+)x(\\d+)[/_].+"
 );
@@ -340,6 +354,11 @@ static Images get_illust_images(const nlohmann::json& image, std::optional<nlohm
     add_if_exists("url");
     if (image.contains("url_big") && image["url_big"].is_string()) {
         images.original = {image["url_big"].get<std::string>(), original_size};
+    } else {
+        std::optional<std::string> original_url = get_original_illust_image(images.thumbnail_or_original().url);
+        if (original_url) {
+            images.original = {std::move(*original_url), original_size};
+        }
     }
 
     if (add_360x360_to >= 0) {

routes/artworks.cpp

@@ -57,7 +57,7 @@ void artworks_route(const httplib::Request& req, httplib::Response& res, const C
 }
 
 static inline Element generate_user_link(const httplib::Request& req, const Config& config, const Illust& illust) {
-    std::string profile_picture = proxy_image_url(config, illust.user_profile_pictures.thumbnail_or_original().url);
+    std::string profile_picture = proxy_image_url(req, config, illust.user_profile_pictures.thumbnail_or_original().url);
     std::string user_link = get_origin(req, config) + "/users/" + std::to_string(illust.user_id);
 
     return Element("a", {{"class", "user_metadata"}, {"href", std::move(user_link)}}, {
@@ -86,12 +86,12 @@ static inline Element generate_images(const httplib::Request& req, const Config&
             div.nodes.push_back(Element("a", {{"class", "landmark"}, {"id", id}, {"href", "#"s + id}}, {id, "/", std::to_string(illust.images.size())}));
         }
 
-        Element img("img", {{"loading", "lazy"}, {"src", proxy_image_url(config, thumbnail.url)}}, {});
+        Element img("img", {{"loading", "lazy"}, {"src", proxy_image_url(req, config, thumbnail.url)}}, {});
         if (thumbnail.size) {
             img.attributes.push_back({"width", std::to_string(thumbnail.size->first)});
             img.attributes.push_back({"height", std::to_string(thumbnail.size->second)});
         }
-        div.nodes.push_back(Element("a", {{"href", proxy_image_url(config, original.url)}}, {
+        div.nodes.push_back(Element("a", {{"href", proxy_image_url(req, config, original.url)}}, {
             std::move(img)
         }));
     }
@@ -110,7 +110,7 @@ static inline Element generate_preview_images(const httplib::Request& req, const
     grid.nodes.reserve(illust.images.size());
     for (size_t i = 0; i < illust.images.size(); i++) {
         const Images& images = illust.images[i];
-        std::string thumbnail = proxy_image_url(config, images.thumbnail_or_original(1).url);
+        std::string thumbnail = proxy_image_url(req, config, images.thumbnail_or_original(1).url);
         std::string link = no_preview_link + '#' + std::to_string(i + 1);
 
         grid.nodes.push_back(Element("a", {{"href", std::move(link)}}, {
@@ -247,7 +247,7 @@ static inline Nodes generate_ogp_nodes(const httplib::Request& req, const Config
     nodes.reserve(nodes.size() + illust.images.size() * 3);
     for (const Images& images : illust.images) {
         const Image& image = images.thumbnail_or_original();
-        nodes.push_back(Element("meta", {{"property", "og:image"}, {"content", proxy_image_url(config, image.url)}}, {}));
+        nodes.push_back(Element("meta", {{"property", "og:image"}, {"content", proxy_image_url(req, config, image.url)}}, {}));
         if (image.size) {
             nodes.push_back(Element("meta", {{"property", "og:image:width"}, {"content", std::to_string(image.size->first)}}, {}));
             nodes.push_back(Element("meta", {{"property", "og:image:height"}, {"content", std::to_string(image.size->second)}}, {}));

routes/users/common.cpp

@@ -5,18 +5,18 @@
 
 static inline Element generate_user_links(const User& user);
 
-Element generate_user_header(const User& user, const Config& config) {
+Element generate_user_header(const User& user, const httplib::Request& req, const Config& config) {
     Element header("header");
     if (user.cover_images) {
-        std::string cover_original = proxy_image_url(config, user.cover_images->original_or_thumbnail().url);
+        std::string cover_original = proxy_image_url(req, config, user.cover_images->original_or_thumbnail().url);
-        std::string cover_thumbnail = proxy_image_url(config, user.cover_images->thumbnail_or_original().url);
+        std::string cover_thumbnail = proxy_image_url(req, config, user.cover_images->thumbnail_or_original().url);
         header.nodes.push_back(Element("a", {{"href", std::move(cover_original)}}, {
             Element("img", {{"class", "user-cover"}, {"loading", "lazy"}, {"src", std::move(cover_thumbnail)}}, {})
         }));
     }
 
-    std::string profile_picture_original = proxy_image_url(config, user.profile_pictures.original_or_thumbnail().url);
+    std::string profile_picture_original = proxy_image_url(req, config, user.profile_pictures.original_or_thumbnail().url);
-    std::string profile_picture_thumbnail = proxy_image_url(config, user.profile_pictures.thumbnail_or_original().url);
+    std::string profile_picture_thumbnail = proxy_image_url(req, config, user.profile_pictures.thumbnail_or_original().url);
     header.nodes.push_back(Element("div", {{"class", "user_metadata"}}, {
         Element("a", {{"href", std::move(profile_picture_original)}}, {
             Element("img", {{"class", "user_profile_picture"}, {"loading", "lazy"}, {"src", std::move(profile_picture_thumbnail)}}, {})

routes/users/common.h

@@ -1,9 +1,10 @@
 #pragma once
 
+#include <httplib/httplib.h>
 #include "../../blankie/serializer.h"
 struct User; // forward declaration from ../../pixivmodels.h
 struct Config; // forward declaration from ../../config.h
 
 using Element = blankie::html::Element;
 
-Element generate_user_header(const User& user, const Config& config);
+Element generate_user_header(const User& user, const httplib::Request& req, const Config& config);

routes/users/illustrations.cpp

@@ -33,7 +33,7 @@ void user_illustrations_route(const httplib::Request& req, httplib::Response& re
     }
 
     Element body("body", {
-        generate_user_header(std::move(user), config),
+        generate_user_header(std::move(user), req, config),
         generate_illusts_pager(req, config, illusts, page, "illusts")
     });
     serve(req, res, config, user.display_name + "'s illustrations", std::move(body), generate_ogp_nodes(req, config, user, page));
@@ -53,10 +53,10 @@ static inline Nodes generate_ogp_nodes(const httplib::Request& req, const Config
         Element("meta", {{"property", "og:url"}, {"content", std::move(url)}}, {})
     });
     if (user.ogp_image) {
-        nodes.push_back(Element("meta", {{"property", "og:image"}, {"content", proxy_image_url(config, *user.ogp_image)}}, {}));
+        nodes.push_back(Element("meta", {{"property", "og:image"}, {"content", proxy_image_url(req, config, *user.ogp_image)}}, {}));
     } else {
         const Image& image = user.profile_pictures.thumbnail_or_original();
-        nodes.push_back(Element("meta", {{"property", "og:image"}, {"content", proxy_image_url(config, image.url)}}, {}));
+        nodes.push_back(Element("meta", {{"property", "og:image"}, {"content", proxy_image_url(req, config, image.url)}}, {}));
         if (image.size) {
             nodes.push_back(Element("meta", {{"property", "og:image:width"}, {"content", std::to_string(image.size->first)}}, {}));
             nodes.push_back(Element("meta", {{"property", "og:image:height"}, {"content", std::to_string(image.size->second)}}, {}));

servehelper.cpp

@@ -64,7 +64,7 @@ void serve_error(const httplib::Request& req, httplib::Response& res, const Conf
     serve(req, res, config, std::move(title), std::move(body));
 }
 
-void serve_redirect(const httplib::Request& req, httplib::Response& res, const Config& config, std::string url) {
+void serve_redirect(const httplib::Request& req, httplib::Response& res, const Config& config, std::string url, bool permanent) {
     using namespace std::string_literals;
 
     Element body("body", {
@@ -72,7 +72,7 @@ void serve_redirect(const httplib::Request& req, httplib::Response& res, const C
         Element("a", {{"href", url}}, {url}),
         "…"
     });
-    res.set_redirect(url);
+    res.set_redirect(url, permanent ? 301 : 302);
     serve(req, res, config, "Redirecting to "s + std::move(url) + "…", std::move(body));
 }
 
@@ -111,11 +111,13 @@ std::string proxy_url(blankie::murl::Url base, blankie::murl::Url url) {
     return base.to_string();
 }
 
-std::string proxy_image_url(const Config& config, blankie::murl::Url url) {
+std::string proxy_image_url(const httplib::Request& req, const Config& config, blankie::murl::Url url) {
-    if (url.is_host_equal("s.pximg.net") || url.is_host_equal("embed.pixiv.net")) {
+    if (!url.is_host_equal("i.pximg.net")) {
         return url.to_string();
     }
-    return proxy_url(config.image_proxy_url, std::move(url));
+    return url.fragment != "guess_extension"
+        ? proxy_url(config.image_proxy_url, std::move(url))
+        : get_origin(req, config) + "/guess_extension/i.pximg.net" + url.path;
 }
 
 bool should_send_304(const httplib::Request& req, uint64_t hash) {
@@ -175,7 +177,7 @@ static inline Element generate_illusts_grid(const httplib::Request& req, const C
 
 static inline Element generate_illusts_grid_item(const httplib::Request& req, const Config& config, const Illust& illust) {
     std::string illust_url = get_origin(req, config) + "/artworks/" + std::to_string(illust.illust_id);
-    std::string image_url = proxy_image_url(config, illust.images[0].thumbnail_or_original(1).url);
+    std::string image_url = proxy_image_url(req, config, illust.images[0].thumbnail_or_original(1).url);
 
     Element div("div", {{"class", "illusts_grid-illust"}}, {
         Element("a", {{"href", illust_url}}, {

servehelper.h

@@ -15,11 +15,11 @@ using Nodes = std::vector<Node>;
 void serve(const httplib::Request& req, httplib::Response& res, const Config& config, std::string title, Element element, Nodes extra_head = {});
 void serve_error(const httplib::Request& req, httplib::Response& res, const Config& config,
         std::string title, std::optional<std::string> subtitle = std::nullopt, std::optional<std::string> info = std::nullopt);
-void serve_redirect(const httplib::Request& req, httplib::Response& res, const Config& config, std::string url);
+void serve_redirect(const httplib::Request& req, httplib::Response& res, const Config& config, std::string url, bool permanent = false);
 
 std::string get_origin(const httplib::Request& req, const Config& config);
 std::string proxy_url(blankie::murl::Url base, blankie::murl::Url url);
-std::string proxy_image_url(const Config& config, blankie::murl::Url url);
+std::string proxy_image_url(const httplib::Request& req, const Config& config, blankie::murl::Url url);
 bool should_send_304(const httplib::Request& req, uint64_t hash);
 
 Element generate_illusts_pager(const httplib::Request& req, const Config& config, const Illusts& illusts, size_t page, const char* id);