Web Scraping Agent with LangChain

from langchain_openai import ChatOpenAI
from langchain_core.tools import tool

@tool
def fetch_url(url: str) -> str:
    """Fetch a webpage and return its content as markdown."""
    import httpx
    from markdownify import markdownify
    response = httpx.get(url, headers={"User-Agent": "Mozilla/5.0"}, timeout=15)
    return markdownify(response.text)[:5000]

@tool
def extract_data(text: str, instruction: str) -> str:
    """Extract structured data from text based on instruction."""
    # Uses the LLM itself to parse — no regex needed
    return f"Extracting from {len(text)} chars: {instruction}"

model = ChatOpenAI(model="gpt-4o")
model_with_tools = model.bind_tools([fetch_url, extract_data])
response = model_with_tools.invoke([
    ("system", "You are a web scraping agent. Fetch pages, extract the requested data, and return it in structured format. Respect robots.txt."),
    ("user", "Scrape the pricing page at example.com/pricing and extract all plan names and prices"),
])