diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000000000000000000000000000000000..f11030d836b8e995b617f38ed5e012bb98857939 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,9 @@ +node_modules +.output +.git +.env +.env.* +.lovable +.vite +dist +*.log diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..d24df8a769e2bd7dd6839f2a51f9e0114d2023c5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,32 @@ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* +lerna-debug.log* + +node_modules +dist +dist-ssr +.output +.vinxi +.tanstack/** +.nitro +*.local + +# Wrangler / Cloudflare +.wrangler/ +.dev.vars + +# Editor directories and files +.vscode/* +!.vscode/extensions.json +.idea +.DS_Store +*.suo +*.ntvs* +*.njsproj +*.sln +*.sw? diff --git a/.lovable/plan.md b/.lovable/plan.md new file mode 100644 index 0000000000000000000000000000000000000000..1aece0a04d8429d0db083a6c08b05a723a96ba30 --- /dev/null +++ b/.lovable/plan.md @@ -0,0 +1,78 @@ +## Goal + +For every image included in the downloaded ZIP, also add: + +- `.txt` — UTF-8 tags file (space-separated, one line, matches the image filename exactly except for extension). +- `.md` — human-readable markdown containing all extracted post metadata (all tags, source link — blank if missing, file size, sample link, original file link, rating, score, dimensions, uploader, created date, md5, etc.). + +## Approach + +The server currently only receives `{id, fileUrl, ext}` per post. To write rich metadata into the ZIP without a second network round-trip per post, the frontend will forward the full normalized post (plus a raw metadata bag) to `/api/download-zip`, and the server will emit the extra files alongside each image. + +To capture "any data related to that post", `searchPosts` will additionally return the raw JSON entry for each post (as `raw: unknown`). The Markdown generator will pretty-print known fields first, then append the full raw JSON as a fenced code block so nothing is lost. + +New optional fields (`sampleUrl`, `fileSize`, `width`, `height`, `md5`, `uploader`, `createdAt`) get added to `SiteConfig.fields` as optional dotted paths, defaulted for the built-in e621 site. Existing custom sites keep working because every new field is optional. + +## Files to change + +### `src/lib/grabber/sites.ts` + +- Extend `FieldMap` with optional: `sampleUrl?`, `fileSize?`, `width?`, `height?`, `md5?`, `uploader?`, `createdAt?`. +- Extend `NormalizedPost` with the same optional fields plus `raw: unknown`. +- Update `normalizePost()` to resolve each new path when present and attach `raw` (the original entry). +- Update `BUILTIN_SITES` e621 entry `fields` with: `sampleUrl: "sample.url"`, `fileSize: "file.size"`, `width: "file.width"`, `height: "file.height"`, `md5: "file.md5"`, `uploader: "uploader_id"`, `createdAt: "created_at"`. + +### `src/lib/grabber/search.functions.ts` + +- Extend the `SiteSchema.fields` zod object with the new optional string paths. +- No handler logic changes needed — normalization already carries `raw` and new fields through. + +### `src/components/grabber/AddSiteDialog.tsx` + +- Add the new optional field keys to the `(["id",...] as const)` list on line 73 so users can map them for custom sites (`sampleUrl`, `fileSize`, `width`, `height`, `md5`, `uploader`, `createdAt`). +- Extend `DEFAULT` (lines 22–31) with sensible Danbooru mappings (`sampleUrl: "large_file_url"`, `fileSize: "file_size"`, `width: "image_width"`, `height: "image_height"`, `md5: "md5"`, `uploader: "uploader_name"`, `createdAt: "created_at"`). + +### `src/routes/api/download-zip.ts` (major rewrite) + +- Expand `BodySchema.posts` item to accept the full normalized post shape (all new optional fields + `tags: string[]` + `raw: unknown`). +- For each post: + 1. Fetch the image bytes as today. Build image filename `base = "_"` and `.`. + 2. Write `.txt` — `strToU8(tags.join(" "))` (UTF-8). Uses the exact same basename as the image so tag files pair 1:1. + 3. Write `.md` — pretty markdown with: + - `# Post ` heading + - Table/list of: Source (blank line if missing), Rating, Score, Dimensions (`width x height`), File size (human-readable KB/MB), MD5, Uploader, Created at, Original file URL, Sample URL, Preview URL, Ext. + - `## Tags` section with the tags space-separated. + - `## Raw metadata` fenced ```json block containing `JSON.stringify(raw, null, 2)`. +- Keep the existing `_errors.txt` behavior and 200-post cap. +- Sanitizer stays the same; md/txt names use `sanitize(id)` so filenames match the image exactly. + +### `src/routes/index.tsx` + +- In `downloadZip()` (the `fetch("/api/download-zip", …)` call), send the full post objects instead of a stripped `{id,fileUrl,ext}` triple. Change: + ``` + posts: chosen.map((p) => ({ id: p.id, fileUrl: p.fileUrl, ext: p.ext })), + ``` + to: + ``` + posts: chosen, // full NormalizedPost incl. tags + raw + optional metadata + ``` +- No other UI changes. + +## ZIP layout example + +```text +grabber-2026-07-17T...zip +├── e621_1234567.jpg +├── e621_1234567.txt ← "tag1 tag2 tag3 ..." +├── 1234567.md ← human-readable metadata + raw JSON +├── e621_1234568.png +├── e621_1234568.txt +├── 1234568.md +└── _errors.txt (only if any fetch failed) +``` + +## Notes / non-goals + +- No new dependencies (still `fflate` + `zod`). +- No DB, no auth, no schema changes — this is presentation + payload shape only. +- Custom sites without the new mappings still work; missing fields render as blank lines in the `.md` and the raw JSON section still contains everything from the API. diff --git a/.lovable/project.json b/.lovable/project.json new file mode 100644 index 0000000000000000000000000000000000000000..22d11d01f809397ac1160f39942580b999f1313f --- /dev/null +++ b/.lovable/project.json @@ -0,0 +1,5 @@ +{ + "schemaVersion": 1, + "template": "tanstack_start_ts_current", + "revision": "tanstack_start_ts_current-fa3af8ca0703" +} diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000000000000000000000000000000000000..be49b2655e2222e8a688f616a5c13500298801a0 --- /dev/null +++ b/.prettierignore @@ -0,0 +1,8 @@ +node_modules +dist +.output +.vinxi +pnpm-lock.yaml +package-lock.json +bun.lock +routeTree.gen.ts diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 0000000000000000000000000000000000000000..90abee2393e79dff7ec36856c94f662a263f0107 --- /dev/null +++ b/.prettierrc @@ -0,0 +1,6 @@ +{ + "printWidth": 100, + "semi": true, + "singleQuote": false, + "trailingComma": "all" +} diff --git a/AGENTS.md b/AGENTS.md index 36eb109884244365c74c416af1644fcdb9bcc9a3..f59511a66eb985d4a2fe4095d2723a65ac52689c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,4 +1,5 @@ + > [!IMPORTANT] > This project is connected to [Lovable](https://lovable.dev). Avoid rewriting > published git history — force pushing, or rebasing/amending/squashing commits @@ -7,4 +8,5 @@ > > Commits you push to the connected branch sync back to Lovable and show up in > the editor, so keep the branch in a working state. + diff --git a/_gitdownloads_grabberintanstack.rar b/_gitdownloads_grabberintanstack.rar index 575f82a8c842ebae648289ff33853a73a350a3ef..d9889a15168690d2270c89f7a462737ef99ef8c6 100644 --- a/_gitdownloads_grabberintanstack.rar +++ b/_gitdownloads_grabberintanstack.rar @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b3bd20f9e6477a94eebd4ef44a7b7b7a34cb085b67dfea3c5948e640e40a7edd -size 429029 +oid sha256:86481e353a158233c09a61b81ee1cfe587d33dca34cc8d17a5cb9248e73c98b9 +size 131 diff --git a/public/favicon.ico b/public/favicon.ico new file mode 100644 index 0000000000000000000000000000000000000000..3c01d69713f9c184e92b74f5799e6dff2f500825 Binary files /dev/null and b/public/favicon.ico differ diff --git a/src/components/grabber/AddSiteDialog.tsx b/src/components/grabber/AddSiteDialog.tsx new file mode 100644 index 0000000000000000000000000000000000000000..ad18e04d4255ba6978bcf91d3c47188cd2637388 --- /dev/null +++ b/src/components/grabber/AddSiteDialog.tsx @@ -0,0 +1,164 @@ +import { useState } from "react"; +import type { SiteConfig } from "@/lib/grabber/sites"; +import { + Dialog, + DialogContent, + DialogHeader, + DialogTitle, + DialogFooter, +} from "@/components/ui/dialog"; +import { Input } from "@/components/ui/input"; +import { Label } from "@/components/ui/label"; +import { Button } from "@/components/ui/button"; +import { Textarea } from "@/components/ui/textarea"; + +type Props = { + open: boolean; + onOpenChange: (v: boolean) => void; + onAdd: (site: SiteConfig) => void; +}; + +const DEFAULT: SiteConfig = { + id: "danbooru", + name: "Danbooru", + baseUrl: "https://danbooru.donmai.us", + searchPath: "/posts.json", + params: { tags: "tags", page: "page", limit: "limit" }, + postsPath: "", + fields: { + id: "id", + previewUrl: "preview_file_url", + fileUrl: "file_url", + ext: "file_ext", + tags: "tag_string_general", + source: "source", + rating: "rating", + score: "score", + sampleUrl: "large_file_url", + fileSize: "file_size", + width: "image_width", + height: "image_height", + md5: "md5", + uploader: "uploader_name", + createdAt: "created_at", + }, +}; + +export function AddSiteDialog({ open, onOpenChange, onAdd }: Props) { + const [s, setS] = useState(DEFAULT); + const [headersText, setHeadersText] = useState(""); + const set = (k: K, v: SiteConfig[K]) => + setS((p) => ({ ...p, [k]: v })); + const setField = (k: keyof SiteConfig["fields"], v: string) => + setS((p) => ({ ...p, fields: { ...p.fields, [k]: v } })); + const setParam = (k: keyof SiteConfig["params"], v: string) => + setS((p) => ({ ...p, params: { ...p.params, [k]: v } })); + + const submit = () => { + let headers: Record | undefined; + if (headersText.trim()) { + try { + headers = JSON.parse(headersText); + } catch { + alert("Headers must be valid JSON"); + return; + } + } + onAdd({ ...s, headers }); + onOpenChange(false); + }; + + return ( + + + + Add Site (schema) + +
+ + set("id", e.target.value)} /> + + + set("name", e.target.value)} /> + + + set("baseUrl", e.target.value)} /> + + + set("searchPath", e.target.value)} /> + + + setParam("tags", e.target.value)} /> + + + setParam("page", e.target.value)} /> + + + setParam("limit", e.target.value)} /> + + + set("postsPath", e.target.value)} /> + +
+ Field mappings (dotted paths) +
+ {( + [ + "id", + "previewUrl", + "fileUrl", + "ext", + "tags", + "source", + "rating", + "score", + "sampleUrl", + "fileSize", + "width", + "height", + "md5", + "uploader", + "createdAt", + ] as const + ).map((k) => ( + + setField(k, e.target.value)} /> + + ))} + + +