Web Scraping Agent with Pydantic AI

from pydantic_ai import Agent, RunContext

agent = Agent(
    "openai:gpt-4o",
    system_prompt="You are a web scraping agent. Fetch pages, extract the requested data, and return it in structured format. Respect robots.txt.",
)

@agent.tool
async def fetch_url(ctx: RunContext, url: str) -> str:
    """Fetch a webpage and return its content as markdown."""
    import httpx
    from markdownify import markdownify
    async with httpx.AsyncClient() as client:
        response = await client.get(url, headers={"User-Agent": "Mozilla/5.0"}, timeout=15)
    return markdownify(response.text)[:5000]

@agent.tool
async def extract_data(ctx: RunContext, text: str, instruction: str) -> str:
    """Extract structured data from text based on instruction."""
    return f"Extracting from {len(text)} chars: {instruction}"

result = await agent.run("Scrape the pricing page at example.com/pricing and extract all plan names and prices")
print(result.output)