Web Scraping Agent with LangGraph

from langchain_openai import ChatOpenAI
from langchain_core.tools import tool
from langgraph.prebuilt import create_react_agent

@tool
def fetch_url(url: str) -> str:
    """Fetch a webpage and return its content as markdown."""
    import httpx
    from markdownify import markdownify
    response = httpx.get(url, headers={"User-Agent": "Mozilla/5.0"}, timeout=15)
    return markdownify(response.text)[:5000]

@tool
def extract_data(text: str, instruction: str) -> str:
    """Extract structured data from text based on instruction."""
    # Uses the LLM itself to parse — no regex needed
    return f"Extracting from {len(text)} chars: {instruction}"

agent = create_react_agent(
    ChatOpenAI(model="gpt-4o"),
    tools=[fetch_url, extract_data],
    prompt="You are a web scraping agent. Fetch pages, extract the requested data, and return it in structured format. Respect robots.txt.",
)

result = await agent.ainvoke({
    "messages": [("user", "Scrape the pricing page at example.com/pricing and extract all plan names and prices")]
})
print(result["messages"][-1].content)