wow
This commit is contained in:
parent
b12ebb725d
commit
5207f5d431
25 changed files with 2932 additions and 332 deletions
|
|
@ -2,202 +2,17 @@ import { Elysia, t } from "elysia";
|
|||
import config from "../config.toml";
|
||||
import * as mongoose from "mongoose";
|
||||
import openapi from "@elysiajs/openapi";
|
||||
import { MediaUpload } from "@/models/media";
|
||||
import { Tag } from "@/models/tag";
|
||||
import { checkTweetData, fetchTweetData } from "./lib/tweet";
|
||||
import { makeS3FileName, s3Client, uploadToS3 } from "./lib/s3";
|
||||
import { normalizeQueryTags, normalizeTags } from "./lib/tag";
|
||||
|
||||
await mongoose.connect(config.mongodb.uri);
|
||||
|
||||
const inFlightUploads = new Set<string>();
|
||||
|
||||
function buildUploadKey(url: string, selected: boolean[]) {
|
||||
const match = url.match(/\/status\/(\d+)/);
|
||||
const tweetId = match?.[1] ?? url;
|
||||
const selectedIndices = selected
|
||||
.map((isSelected, index) => (isSelected ? index : -1))
|
||||
.filter((index) => index >= 0)
|
||||
.join(",");
|
||||
|
||||
return `${tweetId}:${selectedIndices}`;
|
||||
}
|
||||
|
||||
async function saveTags(tags: string[]) {
|
||||
await Promise.all(
|
||||
tags.map((tag) =>
|
||||
Tag.updateOne(
|
||||
{ name: tag },
|
||||
{
|
||||
$inc: { usageCount: 1 },
|
||||
$set: { lastUsedAt: new Date() },
|
||||
$setOnInsert: { name: tag },
|
||||
},
|
||||
{ upsert: true },
|
||||
),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
const app = new Elysia()
|
||||
.use(openapi())
|
||||
.get("/", () => "어...")
|
||||
|
||||
.get("/total", async ({ query }) => {
|
||||
const filterTags = normalizeQueryTags(query.tags);
|
||||
const filter = filterTags.length > 0
|
||||
? { tags: { $in: filterTags } }
|
||||
: {};
|
||||
|
||||
const count = await MediaUpload.countDocuments(filter);
|
||||
return count;
|
||||
}, {
|
||||
query: t.Object({
|
||||
tags: t.Optional(t.Union([t.String(), t.Array(t.String())])),
|
||||
})
|
||||
})
|
||||
|
||||
.get("/list", async ({ query }) => {
|
||||
const page = query.page;
|
||||
const pageSize = 20;
|
||||
const filterTags = normalizeQueryTags(query.tags);
|
||||
|
||||
const filter = filterTags.length > 0
|
||||
? { tags: { $in: filterTags } }
|
||||
: {};
|
||||
|
||||
const uploads = await MediaUpload.find(filter)
|
||||
.sort({ createdAt: -1 })
|
||||
.skip((page - 1) * pageSize)
|
||||
.limit(pageSize);
|
||||
return uploads;
|
||||
}, {
|
||||
query: t.Object({
|
||||
page: t.Number({default: 1, minimum: 1}),
|
||||
tags: t.Optional(t.Union([t.String(), t.Array(t.String())])),
|
||||
})
|
||||
})
|
||||
|
||||
.get("/tags", async () => {
|
||||
const tags = await Tag.find().sort({ usageCount: -1, lastUsedAt: -1 });
|
||||
return tags;
|
||||
})
|
||||
|
||||
.post("/upload", async ({ body, status }) => {
|
||||
if (body.url.startsWith("https://www.pixiv.net/")) {
|
||||
return "저는 저능아입니다";
|
||||
} else if (body.url.startsWith("https://x.com/") || body.url.startsWith("https://twitter.com/") || body.url.startsWith("https://fxtwitter.com/") || body.url.startsWith("https://fixupx.com/") || body.url.startsWith("https://vxwitter.com/")) {
|
||||
const requestId = crypto.randomUUID();
|
||||
const uploadKey = buildUploadKey(body.url, body.selected);
|
||||
|
||||
if (inFlightUploads.has(uploadKey)) {
|
||||
console.warn(`[Upload skipped-duplicate] requestId=${requestId} key=${uploadKey}`);
|
||||
return status(202, "이미 처리 중인 업로드입니다.");
|
||||
}
|
||||
|
||||
inFlightUploads.add(uploadKey);
|
||||
console.log(`[Upload started] requestId=${requestId} key=${uploadKey}`);
|
||||
|
||||
if (await checkTweetData(body.url, body.selected)) {
|
||||
inFlightUploads.delete(uploadKey);
|
||||
console.log(`[Upload skipped-existing] requestId=${requestId} key=${uploadKey}`);
|
||||
return "이미 저장된 트윗입니다.";
|
||||
}
|
||||
try {
|
||||
const tweetData = await fetchTweetData(body.url);
|
||||
if (tweetData.tweet) {
|
||||
const media = tweetData.tweet.media.photos || [];
|
||||
if (media.length > 0) {
|
||||
const mediaUrls = media.map((m: any) => m.url);
|
||||
// Upload to S3
|
||||
|
||||
let savedCount = 0;
|
||||
let failedCount = 0;
|
||||
const hasExplicitSelection = body.selected.length > 0;
|
||||
for (const [index, url] of mediaUrls.entries()) {
|
||||
const isSelected = hasExplicitSelection
|
||||
? body.selected[index] === true
|
||||
: true;
|
||||
|
||||
if (!isSelected) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const fileName = makeS3FileName(tweetData.tweet.author.id, tweetData.tweet.id, url, index);
|
||||
|
||||
try {
|
||||
if (await s3Client.exists(fileName)) {
|
||||
console.log(`File ${fileName} already exists in S3, skipping upload.`);
|
||||
} else {
|
||||
await uploadToS3(fileName, url);
|
||||
console.log(`Uploaded ${fileName} to S3`);
|
||||
}
|
||||
|
||||
const { media: _media, ...tweetWithoutMedia } = tweetData.tweet;
|
||||
const normalizedTags = normalizeTags(body.tag || ["미분류"]);
|
||||
|
||||
await MediaUpload.create({
|
||||
type: "twitter",
|
||||
tweet: tweetWithoutMedia,
|
||||
mediaIndex: index,
|
||||
mediaUrl: `${config.s3.endpoint}/${config.s3.bucket}/${fileName}`,
|
||||
s3Key: fileName,
|
||||
tags: normalizedTags,
|
||||
author: body.author ? body.author : tweetData.tweet.author.name,
|
||||
});
|
||||
|
||||
await saveTags(normalizedTags);
|
||||
savedCount += 1;
|
||||
} catch (error) {
|
||||
failedCount += 1;
|
||||
console.error(`[Upload failed] index=${index} url=${url} key=${fileName}`, error);
|
||||
}
|
||||
}
|
||||
|
||||
if (savedCount === 0 && failedCount === 0) {
|
||||
console.warn("No media uploaded: selected[] did not include any upload target.");
|
||||
}
|
||||
|
||||
console.log(`Saved ${savedCount} media records to MongoDB. Failed: ${failedCount}`);
|
||||
} else {
|
||||
console.log("No media found in the tweet.");
|
||||
}
|
||||
}
|
||||
console.log(`[Upload finished] requestId=${requestId} key=${uploadKey}`);
|
||||
console.log(tweetData);
|
||||
} catch (error) {
|
||||
console.error(`[Upload aborted] requestId=${requestId} key=${uploadKey}`, error);
|
||||
console.error(error);
|
||||
return status(500, "Failed to fetch tweet data");
|
||||
} finally {
|
||||
inFlightUploads.delete(uploadKey);
|
||||
}
|
||||
} else {
|
||||
return status(400, "어...");
|
||||
}
|
||||
return "아...";
|
||||
}, {
|
||||
body: t.Object({
|
||||
url: t.String(),
|
||||
tag: t.Optional(t.Array(t.String({default: "미분류"}))),
|
||||
author: t.Optional(t.String()),
|
||||
selected: t.Array(t.Boolean()),
|
||||
})
|
||||
})
|
||||
|
||||
.get("/fetch/tweet", async ({ query, status }) => {
|
||||
try {
|
||||
const tweetData = await fetchTweetData(query.url);
|
||||
return tweetData;
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
return status(500, "Failed to fetch tweet data");
|
||||
}
|
||||
}, {
|
||||
query: t.Object({
|
||||
url: t.String(),
|
||||
})
|
||||
})
|
||||
.use(import("./routes/tweet"))
|
||||
.use(import("./routes/auth"))
|
||||
.use(import("./routes/post"))
|
||||
.use(import("./routes/pixiv"))
|
||||
|
||||
.listen(config.server.port)
|
||||
;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue