Merge pull request 'wow' (#1) from test/new-pixiv-api into main

Reviewed-on: #1

apps/backend/src/lib/pixiv.ts

@@ -1,44 +1,148 @@
 import { MediaUpload } from "@/models/media";
 
-function fetchPixivData(url: string): Promise<any> {
+type PixivTag = {
-    // https://www.pixiv.net/artworks/143552616
+    tag?: unknown;
-    const match = url.match(/\/artworks\/(\d+)/);
+    translation?: { en?: unknown } | null;
+};
+
+type PixivIllustBody = {
+    illustId?: unknown;
+    id?: unknown;
+    illustTitle?: unknown;
+    title?: unknown;
+    illustComment?: unknown;
+    description?: unknown;
+    urls?: Record<string, unknown>;
+    tags?: { tags?: PixivTag[] } | PixivTag[] | string[];
+    userId?: unknown;
+    userName?: unknown;
+    createDate?: unknown;
+    uploadDate?: unknown;
+    pageCount?: unknown;
+    xRestrict?: unknown;
+    url?: unknown;
+};
+
+type PixivAjaxResponse<T> = {
+    error?: boolean;
+    message?: string;
+    body?: T;
+};
+
+type PixivPage = {
+    urls?: Record<string, unknown>;
+};
+
+function extractArtworkId(url: string): string {
+    const match = url.match(/(?:\/artworks\/|\/ajax\/illust\/)(\d+)/);
     if (!match) {
         throw new Error("Invalid Pixiv URL");
     }
-    const artworkId = match[1];
 
+    return match[1];
+}
 
-    return fetch(`https://www.phixiv.net/api/info?id=${artworkId}&language=ko`)
+function pixivAjaxUrl(path: string): string {
-        .then((response) => {
+    return `https://www.pixiv.net/ajax/illust/${path}`;
-            if (!response.ok) {
+}
-                throw new Error(`Failed to fetch Pixiv data: ${response.status} ${response.statusText}`);
+
-            }
+function getPreferredImageUrl(urls: Record<string, unknown> | undefined): string | null {
-            return response.json();
+    if (!urls) return null;
+
+    for (const key of ["original", "regular", "small", "thumb"]) {
+        const value = urls[key];
+        if (typeof value === "string" && value.length > 0) return value;
+    }
+
+    return null;
+}
+
+function normalizeTags(tags: PixivIllustBody["tags"]): string[] {
+    const tagList = Array.isArray(tags)
+        ? tags
+        : tags && typeof tags === "object" && "tags" in tags
+            ? tags.tags
+            : [];
+    if (!Array.isArray(tagList)) return [];
+
+    return tagList
+        .map((entry) => {
+            if (typeof entry === "string") return entry;
+            if (typeof entry.tag === "string") return entry.tag;
+            return null;
         })
-        .then((data) => {
+        .filter((tag): tag is string => tag !== null && tag.length > 0);
-            // if #R-18 in tags, throw error
+}
-            if (data.tags && data.tags.some((tag: string) => tag.includes("R-18"))) {
-                throw new Error("Pixiv artwork is marked as R-18");
-            }
 
-            return data;
+function makePixivRequest(url: string) {
-        })
+    return fetch(url, {
-        .then((data) => {
+        headers: {
-            if (data.error) {
+            "Accept": "application/json",
-                throw new Error(`Pixiv API error: ${data.message}`);
+            "Referer": "https://www.pixiv.net/",
-            }
+            "User-Agent": "Mozilla/5.0 (compatible; akiyama.mizuki.guru/1.0)",
-            return data;
+        },
-        });
+    });
+}
+
+async function readPixivJson<T>(url: string): Promise<T> {
+    const response = await makePixivRequest(url);
+    if (!response.ok) {
+        throw new Error(`Failed to fetch Pixiv data: ${response.status} ${response.statusText}`);
+    }
+
+    const data = (await response.json()) as PixivAjaxResponse<T>;
+    if (data.error) {
+        throw new Error(`Pixiv API error: ${data.message || "unknown error"}`);
+    }
+    if (data.body === undefined) {
+        throw new Error("Pixiv API response did not include a body");
+    }
+
+    return data.body;
+}
+
+async function fetchPixivData(url: string): Promise<any> {
+    const artworkId = extractArtworkId(url);
+    const body = await readPixivJson<PixivIllustBody>(pixivAjaxUrl(`${artworkId}?lang=ko`));
+    const pageCount = typeof body.pageCount === "number" ? body.pageCount : 1;
+    const tags = normalizeTags(body.tags);
+
+    if (tags.some((tag) => tag.includes("R-18"))) {
+        throw new Error("Pixiv artwork is marked as R-18");
+    }
+    if (body.xRestrict === 1 || body.xRestrict === 2) {
+        throw new Error("Pixiv artwork is restricted");
+    }
+
+    let imageUrls: string[] = [];
+    if (pageCount > 1) {
+        const pages = await readPixivJson<PixivPage[]>(pixivAjaxUrl(`${artworkId}/pages?lang=ko`));
+        imageUrls = pages
+            .map((page) => getPreferredImageUrl(page.urls))
+            .filter((imageUrl): imageUrl is string => imageUrl !== null);
+    } else {
+        const imageUrl = getPreferredImageUrl(body.urls);
+        if (imageUrl) imageUrls = [imageUrl];
+    }
+
+    return {
+        url: `https://www.pixiv.net/artworks/${artworkId}`,
+        illust_id: String(body.illustId || body.id || artworkId),
+        title: String(body.illustTitle || body.title || "Pixiv Artwork"),
+        description: String(body.illustComment || body.description || ""),
+        author_id: String(body.userId || "unknown"),
+        author_name: String(body.userName || "unknown"),
+        create_date: typeof body.createDate === "string" ? body.createDate : body.uploadDate,
+        tags,
+        image_proxy_urls: imageUrls,
+        image_urls: imageUrls,
+        page_count: pageCount,
+        raw: body,
+    };
 }
 
 async function checkPixivData(url: string, selected: Array<boolean>) {
-    const match = url.match(/\/artworks\/(\d+)/);
+    const artworkId = extractArtworkId(url);
-    if (!match) {
-        throw new Error("Invalid Pixiv URL");
-    }
-
-    const artworkId = match[1];
     const selectedIndices = selected
         .map((isSelected, index) => (isSelected ? index : -1))
         .filter((index) => index >= 0);
@@ -55,4 +159,4 @@ async function checkPixivData(url: string, selected: Array<boolean>) {
     return existing !== null;
 }
 
-export { checkPixivData, fetchPixivData };
+export { checkPixivData, fetchPixivData };

apps/backend/src/lib/s3.ts

@@ -31,7 +31,16 @@ async function uploadToS3(fileName: string, mediaUrl: string, maxRetry = 3) {
 
   for (let attempt = 1; attempt <= maxRetry; attempt++) {
     try {
-      const response = await fetch(mediaUrl);
+      const parsedUrl = new URL(mediaUrl);
+      const isPixivImage = parsedUrl.hostname === "i.pximg.net";
+      const response = await fetch(mediaUrl, {
+        headers: isPixivImage
+          ? {
+            "Referer": "https://www.pixiv.net/",
+            "User-Agent": "Mozilla/5.0 (compatible; akiyama.mizuki.guru/1.0)",
+          }
+          : undefined,
+      });
       if (!response.ok) {
         throw new Error(`Fetch failed: ${response.status}`);
       }
@@ -74,4 +83,4 @@ async function writeToS3(fileName: string, data: ArrayBuffer | Blob, contentType
   });
 }
 
-export { makeS3FileName, uploadToS3, writeToS3, client as s3Client };
+export { makeS3FileName, uploadToS3, writeToS3, client as s3Client };

apps/backend/src/routes/proxy.ts

@@ -5,6 +5,7 @@ const ALLOWED_HOSTS = [
     "video.twimg.com",
     "ton.twimg.com",
     "abs.twimg.com",
+    "i.pximg.net",
 ];
 
 export default new Elysia({ prefix: "/proxy" })
@@ -25,7 +26,7 @@ export default new Elysia({ prefix: "/proxy" })
         const response = await fetch(targetUrl, {
             headers: {
                 "User-Agent": "Mozilla/5.0 (compatible; akiyama.mizuki.guru/1.0)",
-                "Referer": "https://twitter.com/",
+                "Referer": parsed.hostname === "i.pximg.net" ? "https://www.pixiv.net/" : "https://twitter.com/",
             },
         });
 

apps/frontend/src/app/add/page.tsx

@@ -250,7 +250,7 @@ export default function AddPage() {
             const data = (await response.json()) as PixivApiResponse;
             const items = (data.image_proxy_urls ?? [])
                 .filter((imageUrl): imageUrl is string => typeof imageUrl === "string" && imageUrl.length > 0)
-                .map((imageUrl) => ({ url: imageUrl }));
+                .map((imageUrl) => ({ url: proxyMediaUrl(imageUrl) }));
 
             if (items.length === 0) throw new Error("이미지를 찾지 못했습니다.");
 

apps/frontend/src/lib/media.ts

@@ -3,6 +3,7 @@ const PROXIED_HOSTS = [
   "video.twimg.com",
   "ton.twimg.com",
   "abs.twimg.com",
+  "i.pximg.net",
 ];
 
 export function proxyMediaUrl(url: string | undefined | null): string {

bun.lock

@@ -9,11 +9,11 @@
       "name": "elysia",
       "version": "1.0.50",
       "dependencies": {
-        "@aws-sdk/client-s3": "^3.1053.0",
+        "@aws-sdk/client-s3": "^3.1075.0",
         "@elysiajs/jwt": "^1.4.2",
         "@elysiajs/openapi": "^1.4.15",
         "elysia": "latest",
-        "mongoose": "^9.6.2",
+        "mongoose": "^9.7.2",
         "ulid": "^3.0.2",
       },
       "devDependencies": {