Web Scraping Agent with CrewAI

from crewai import Agent, Crew, Task
from langchain_openai import ChatOpenAI
from langchain_core.tools import tool

@tool
def fetch_url(url: str) -> str:
    """Fetch a webpage and return its content as markdown."""
    import httpx
    from markdownify import markdownify
    response = httpx.get(url, headers={"User-Agent": "Mozilla/5.0"}, timeout=15)
    return markdownify(response.text)[:5000]

@tool
def extract_data(text: str, instruction: str) -> str:
    """Extract structured data from text based on instruction."""
    # Uses the LLM itself to parse — no regex needed
    return f"Extracting from {len(text)} chars: {instruction}"

agent = Agent(
    role="Specialist",
    goal="You are a web scraping agent. Fetch pages, extract the requested data, and return it in structured format. Respect robots.txt.",
    tools=[fetch_url, extract_data],
    llm=ChatOpenAI(model="gpt-4o"),
)

task = Task(
    description="Scrape the pricing page at example.com/pricing and extract all plan names and prices",
    expected_output="Detailed response",
    agent=agent,
)

crew = Crew(agents=[agent], tasks=[task])
result = crew.kickoff()
print(result.raw)