Web Scraping Agent with Deep Agents

from deepagents import create_deep_agent
from langchain_core.tools import tool

@tool
def fetch_url(url: str) -> str:
    """Fetch a webpage and return its content as markdown."""
    import httpx
    from markdownify import markdownify
    response = httpx.get(url, headers={"User-Agent": "Mozilla/5.0"}, timeout=15)
    return markdownify(response.text)[:5000]

@tool
def extract_data(text: str, instruction: str) -> str:
    """Extract structured data from text based on instruction."""
    # Uses the LLM itself to parse — no regex needed
    return f"Extracting from {len(text)} chars: {instruction}"

agent = create_deep_agent(
    model="anthropic:claude-sonnet-4-5-20250929",
    tools=[fetch_url, extract_data],
    system_prompt="You are a web scraping agent. Fetch pages, extract the requested data, and return it in structured format. Respect robots.txt.",
)

result = agent.invoke({
    "messages": [("user", "Scrape the pricing page at example.com/pricing and extract all plan names and prices")]
})
print(result["messages"][-1].content)