วิธีสร้าง AI Web Scraper แบบง่ายด้วย Python เพื่อการดึงข้อมูลที่แม่นยำ

!pip install requests beautifulsoup4 markdownify openai ftfy python-dotenv
``` โดยแต่ละเครื่องมือมีหน้าที่ดังนี้:
 
- **[requests](https://pypi.org/project/requests/)** สำหรับการดึงข้อมูลหน้าเว็บ
- **[BeautifulSoup](https://www.crummy.com/software/BeautifulSoup/)** สำหรับลบองค์ประกอบ HTML ที่ไม่จำเป็นออก
- **[markdownify](https://pypi.org/project/markdownify/)** สำหรับแปลง HTML ให้เป็นรูปแบบ Markdown
- **[OpenAI](https://pypi.org/project/openai/)** สำหรับประมวลผลคำตอบ
- **[ftfy](https://pypi.org/project/ftfy/)** สำหรับแก้ไขปัญหาการแสดงผลตัวอักษรที่ผิดเพี้ยน
- **[python-dotenv](https://pypi.org/project/python-dotenv/)** สำหรับการจัดการ API Key อย่างปลอดภัย
 
จากนั้นทำการ Import ไลบรารีที่ต้องใช้ใน Notebook:
 
```python
import os
import re
import requests
 
from bs4 import BeautifulSoup, Comment
from ftfy import fix_text
from markdownify import markdownify as markdownify_html
from openai import OpenAI
from dotenv import load_dotenv
from IPython.display import Markdown, display

OPENAI_API_KEY=your_api_key_here

load_dotenv()
 
client = OpenAI(api_key=os.getenv("OPENAI_API_KEY"))

MODEL_NAME = "gpt-5.4-nano"

def fetch_page(url: str) -> str:
    """
    Download the HTML content from a webpage.
    """
    headers = {
        "User-Agent": "SimpleAIScraper/1.0"
    }
 
response = requests.get(url, headers=headers, timeout=15)
    response.raise_for_status()
 
return response.text

raw = fetch_page("https://www.olostep.com/")
print(raw[:500])

def clean_html(html):
    html = fix_text(html)
 
soup = BeautifulSoup(html, "html.parser")
 
# Remove obvious noisy tags
    for tag in soup([
        "script", "style", "noscript", "svg", "img", "iframe",
        "nav", "header", "footer", "aside", "form", "button"
    ]):
        tag.decompose()
 
noise_words = [
        "cursor", "modal", "popup", "floating", "signup", "login",
        "cookie", "banner", "navbar", "menu", "footer", "header",
        "subscribe", "newsletter", "loading", "wait", "success",
        "auth", "w-nav", "w-form"
    ]
 
# First collect noisy tags
    tags_to_remove = []
 
for tag in soup.find_all(True):
        if tag.attrs is None:
            continue
 
class_value = tag.get("class", [])
        id_value = tag.get("id", "")
 
if isinstance(class_value, list):
            class_text = " ".join(class_value).lower()
        else:
            class_text = str(class_value).lower()
 
id_text = str(id_value).lower()
 
if any(word in class_text or word in id_text for word in noise_words):
            tags_to_remove.append(tag)
 
# Then remove them safely
    for tag in tags_to_remove:
        tag.decompose()
 
body = soup.body if soup.body else soup
 
return str(body)

clean = clean_html(raw)
print(clean[:500])

def html_to_markdown(html):
    markdown_text = markdownify_html(
        html,
        heading_style="ATX",
        bullets="-"
    )
 
markdown_text = fix_text(markdown_text)
 
# Remove image markdown
    markdown_text = re.sub(r"!\[.*?\]\(.*?\)", "", markdown_text)
 
# Remove extra spaces and blank lines
    markdown_text = re.sub(r"[ \t]+", " ", markdown_text)
    markdown_text = re.sub(r"\n{3,}", "\n\n", markdown_text)
 
lines = []
 
skip_lines = [
        "click to try", "wait...", "you've successfully reserved your spot.",
        "thank you! your submission has been received!",
        "oops! something went wrong while submitting the form.",
        "product", "resources", "company"
    ]
 
for line in markdown_text.splitlines():
        line = line.strip()
 
if not line:
            continue
 
if line.lower() in skip_lines:
            continue
 
lines.append(line)
 
return "\n".join(lines)

md = html_to_markdown(clean)
print(md[:500])

def answer_query_from_page(markdown_text, user_query):
    prompt = f"""
You are an AI web scraping assistant.
You will receive Markdown extracted from a webpage.
Your task is to answer the user's query using only the useful page content.
 
User query:
{user_query}
 
Webpage Markdown:
{markdown_text}
 
Instructions:
- Return only clean Markdown.
- Use only information from the webpage Markdown.
- Do not invent missing details.
- Ignore navigation links, buttons, CTAs, popups, decorative labels, image captions, and repeated marketing fragments.
- Ignore lines like "Start for free", "Contact Sales", "Your AI Agent", and decorative workflow examples unless they directly answer the query.
- Focus on headings, paragraphs, product descriptions, feature sections, pricing details, documentation text, and factual claims.
- If the page does not contain the answer, say: "The page does not contain this information."
- Keep the answer short, clear, and focused.
"""
 
response = client.responses.create(
        model=MODEL_NAME,
        input=prompt
    )
 
return response.output_text

def ai_web_scraper(url, user_query):
    raw_html = fetch_page(url)
    cleaned_html = clean_html(raw_html)
    markdown_text = html_to_markdown(cleaned_html)
    answer = answer_query_from_page(markdown_text, user_query)
 
return answer

url = "https://www.olostep.com/"
user_query = "What does this company do?"
result = ai_web_scraper(url, user_query)
 
display(Markdown(result))

url = "https://www.olostep.com/pricing"
user_query = "Help me understand the pricing"
result = ai_web_scraper(url, user_query)
 
display(Markdown(result))

with open("ai_scraper_result.md", "w", encoding="utf-8") as file:
    file.write(result)
 
print("Markdown saved to ai_scraper_result.md")