Implement HTML domain checker in Rust and add PowerShell script for chunked execution

- Added a Rust program that reads a list of domains from "domains.txt", checks if they return HTML content, and writes valid domains to "domains-filtered.txt".
- Introduced a PowerShell script to execute a Python script in chunks, allowing for processing of specified line ranges from the domain list.

is-html-fast/.gitignore

@@ -0,0 +1,71 @@
+# Generated by Cargo
+# will have compiled files and executables
+debug/
+target/
+
+# These are backup files generated by rustfmt
+**/*.rs.bk
+
+# MSVC Windows builds of rustc generate these, which store debugging information
+*.pdb
+
+# Generated by cargo mutants
+# Contains mutation testing data
+**/mutants.out*/
+
+# RustRover
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+# General
+.DS_Store
+.AppleDouble
+.LSOverride
+Icon[]
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+# Windows thumbnail cache files
+Thumbs.db
+Thumbs.db:encryptable
+ehthumbs.db
+ehthumbs_vista.db
+
+# Dump file
+*.stackdump
+
+# Folder config file
+[Dd]esktop.ini
+
+# Recycle Bin used on file shares
+$RECYCLE.BIN/
+
+# Windows Installer files
+*.cab
+*.msi
+*.msix
+*.msm
+*.msp
+
+# Windows shortcuts
+*.lnk

is-html-fast/Cargo.toml

@@ -0,0 +1,8 @@
+[package]
+name = "is-html-fast"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+rayon = "1.10.0"
+reqwest = { version = "0.12.19", features = ["blocking", "json"]}

is-html-fast/README.md

@@ -0,0 +1,2 @@
+실제로 사용되진 않습니다.
+일회용 코드입니다.

is-html-fast/src/main.rs

@@ -0,0 +1,59 @@
+use std::fs::{File, OpenOptions};
+use std::io::{BufRead, BufReader, Write};
+use std::sync::{Arc, Mutex};
+use std::time::Duration;
+
+use rayon::prelude::*;
+use reqwest::blocking::Client;
+use reqwest::header::CONTENT_TYPE;
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let input_file = File::open("domains.txt")?;
+    let reader = BufReader::new(input_file);
+    let domains: Vec<String> = reader.lines().filter_map(Result::ok).collect();
+
+    let output_file = OpenOptions::new()
+        .create(true)
+        .write(true)
+        .truncate(true)
+        .open("domains-filtered.txt")?;
+
+    let output = Arc::new(Mutex::new(output_file));
+
+    let client = Arc::new(
+        Client::builder()
+            .timeout(Duration::from_secs(5))
+            .build()?,
+    );
+
+    domains.par_iter().for_each(|domain| {
+        let url = format!("https://{}", domain);
+        println!("Checking {}", url);
+
+        let response = client.get(&url).send();
+
+        match response {
+            Ok(resp) => {
+                if let Some(content_type) = resp.headers().get(CONTENT_TYPE) {
+                    if let Ok(content_type_str) = content_type.to_str() {
+                        if content_type_str.starts_with("text/html") {
+                            if let Ok(mut file) = output.lock() {
+                                writeln!(file, "{}", domain).ok();
+                            }
+                            println!("✅ HTML: {}", domain);
+                        } else {
+                            println!("❌ Not HTML: {} ({})", domain, content_type_str);
+                        }
+                    }
+                } else {
+                    println!("❌ No Content-Type: {}", domain);
+                }
+            }
+            Err(_) => {
+                println!("⚠️ Failed to connect: {}", domain);
+            }
+        }
+    });
+
+    Ok(())
+}

main.py

@@ -100,9 +100,9 @@ extend_planner_system_message = """
 """
 
 # ── URL별로 Browser를 새로 띄우는 함수 ──
-async def scan_one_url(url: str):
+async def scan_one_url(url: str, skip_html_check: bool = False):
     # 1) URL이 HTML 페이지인지 확인
-    if not is_html_url(url):
+    if not is_html_url(url) and not skip_html_check:
         print(f"❌ {url} 은(는) HTML이 아닙니다. 스킵합니다.")
         return
 
@@ -181,7 +181,7 @@ async def scan_one_url(url: str):
         await context.close()
         await browser.close()
 
-async def loop(filepath: str, start_line: int, end_line: int):
+async def loop(filepath: str, start_line: int, end_line: int, skip_html_check: bool = False):
     # 인자값으로 받은 파일 경로와 줄 범위를 통해 도메인 리스트 생성
     target_list = read_lines_between(
         filepath=filepath,
@@ -196,7 +196,7 @@ async def loop(filepath: str, start_line: int, end_line: int):
         # scan_one_url은 외부에 정의된 비동기 함수라고 가정합니다.
         # 실제로 scan_one_url이 정의된 위치를 import하거나
         # 모듈 수준에 구현해두셔야 합니다.
-        await scan_one_url(f'https://{url}')
+        await scan_one_url(f'https://{url}', skip_html_check=skip_html_check)
 
 
 def main():
@@ -224,6 +224,12 @@ def main():
         required=True,
         help="읽기 종료 줄 번호 (1-based)"
     )
+    parser.add_argument(
+        "-skh", "--skip-html-check",
+        type=bool,
+        default=False,
+        help="HTML 페이지 체크를 건너뛰고 모든 URL을 스캔합니다. (기본값: False)"
+    )
 
     args = parser.parse_args()
 
@@ -231,7 +237,8 @@ def main():
     asyncio.run(loop(
         filepath=args.file,
         start_line=args.start,
-        end_line=args.end
+        end_line=args.end,
+        skip_html_check=args.skip_html_check
     ))
 
 

run.ps1

@@ -0,0 +1,47 @@
+# ── 설정 부분 ──
+# 실행할 Python 스크립트 이름 (파일 확장자까지)
+$PYTHON_SCRIPT = "main.py"
+
+# 도메인 목록 파일 경로 (Python 스크립트 실행 시 -f 옵션에 전달)
+$DOMAIN_FILE = "./domains.txt"
+
+# 몇 줄씩(chunk) 나눠서 실행할지
+$CHUNK_SIZE = 10
+# ─────────────
+
+# 인자 개수 확인 (2개 또는 3개)
+if ($args.Count -lt 2 -or $args.Count -gt 3) {
+    Write-Host "Usage: $($MyInvocation.MyCommand.Name) <start_line> <end_line> [skip_header]"
+    Write-Host "예시) $($MyInvocation.MyCommand.Name) 10000 11000"
+    Write-Host "예시) $($MyInvocation.MyCommand.Name) 10000 11000 True"
+    exit 1
+}
+
+$START_LINE = [int]$args[0]
+$END_LINE = [int]$args[1]
+$SKIP_HEADER = if ($args.Count -eq 3) { $args[2] } else { "False" }
+
+# START_LINE부터 END_LINE까지 CHUNK_SIZE 만큼씩 반복
+$current = $START_LINE
+while ($current -le $END_LINE) {
+    # 각 청크 구간의 마지막 줄 계산
+    $chunk_end = $current + $CHUNK_SIZE - 1
+    if ($chunk_end -gt $END_LINE) {
+        $chunk_end = $END_LINE
+    }
+
+    $timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss"
+    Write-Host "[$timestamp] Processing lines $current to $chunk_end..."
+    
+    # Python 스크립트 실행
+    # -f DOMAIN_FILE: 도메인 목록 파일 경로
+    # -s current  : 읽기 시작 줄
+    # -e chunk_end: 읽기 끝 줄
+    # -skh SKIP_HEADER: 헤더 스킵 여부
+    uv run $PYTHON_SCRIPT -f $DOMAIN_FILE -s $current -e $chunk_end -skh $SKIP_HEADER
+
+    # 다음 청크의 시작 값 설정
+    $current = $chunk_end + 1
+}
+
+Write-Host "모든 청크 처리 완료."

run.sh

@@ -35,7 +35,8 @@ while [ "$current" -le "$END_LINE" ]; do
   # -f DOMAIN_FILE: 도메인 목록 파일 경로
   # -s current  : 읽기 시작 줄
   # -e chunk_end: 읽기 끝 줄
-  uv run "$PYTHON_SCRIPT" -f "$DOMAIN_FILE" -s "$current" -e "$chunk_end"
+  # -skh True False: 추가 옵션
+  uv run "$PYTHON_SCRIPT" -f "$DOMAIN_FILE" -s "$current" -e "$chunk_end" -skh $3
 
   # 다음 청크의 시작 값 설정
   current=$(( chunk_end + 1 ))