[{"data":1,"prerenderedAt":662},["ShallowReactive",2],{"navigation_docs":3,"-guide-agents":68,"-guide-agents-surround":657},[4,32],{"title":5,"path":6,"stem":7,"children":8,"icon":31},"Guide","\u002Fguide","1.guide\u002F1.index",[9,11,15,19,23,27],{"title":10,"path":6,"stem":7},"Getting Started",{"title":12,"path":13,"stem":14},"Discovering URLs","\u002Fguide\u002Fdiscover","1.guide\u002F2.discover",{"title":16,"path":17,"stem":18},"Filters and scope","\u002Fguide\u002Ffilters","1.guide\u002F3.filters",{"title":20,"path":21,"stem":22},"CLI","\u002Fguide\u002Fcli","1.guide\u002F4.cli",{"title":24,"path":25,"stem":26},"Agents","\u002Fguide\u002Fagents","1.guide\u002F5.agents",{"title":28,"path":29,"stem":30},"Custom providers","\u002Fguide\u002Fcustom","1.guide\u002F6.custom","i-lucide-book-open",{"title":33,"path":34,"stem":35,"children":36,"icon":67},"Providers","\u002Fproviders","2.providers\u002F0.index",[37,39,43,47,51,55,59,63],{"title":38,"path":34,"stem":35},"Overview",{"title":40,"path":41,"stem":42},"AlienVault OTX","\u002Fproviders\u002Falienvault","2.providers\u002F01.alienvault",{"title":44,"path":45,"stem":46},"Arquivo.pt","\u002Fproviders\u002Farquivo","2.providers\u002F02.arquivo",{"title":48,"path":49,"stem":50},"Common Crawl","\u002Fproviders\u002Fcommoncrawl","2.providers\u002F03.commoncrawl",{"title":52,"path":53,"stem":54},"URLScan","\u002Fproviders\u002Furlscan","2.providers\u002F04.urlscan",{"title":56,"path":57,"stem":58},"Vefsafn","\u002Fproviders\u002Fvefsafn","2.providers\u002F05.vefsafn",{"title":60,"path":61,"stem":62},"VirusTotal","\u002Fproviders\u002Fvirustotal","2.providers\u002F06.virustotal",{"title":64,"path":65,"stem":66},"Wayback Machine","\u002Fproviders\u002Fwayback","2.providers\u002F07.wayback","i-lucide-library",{"id":69,"title":24,"body":70,"description":649,"extension":650,"links":651,"meta":652,"navigation":258,"path":25,"seo":654,"stem":26,"__hash__":656},"docs\u002F1.guide\u002F5.agents.md",{"type":71,"value":72,"toc":640},"minimark",[73,86,91,188,211,215,305,312,316,347,350,354,361,364,423,447,452,595,609,620,624,636],[74,75,76,77,81,82,85],"p",{},"Every agent host gets the same two tools, backed by the same executor: ",[78,79,80],"code",{},"urls_discover"," and ",[78,83,84],{},"urls_providers",". What changes between hosts is only how the answer is printed.",[87,88,90],"h2",{"id":89},"mcp","MCP",[92,93,99],"pre",{"className":94,"code":95,"filename":96,"language":97,"meta":98,"style":98},"language-json shiki shiki-themes urls urls urls","{\n  \"mcpServers\": {\n    \"urls\": {\n      \"command\": \"npx\",\n      \"args\": [\"-y\", \"@agntn\u002Furls\", \"mcp\"]\n    }\n  }\n}\n","mcp.json","json","",[78,100,101,110,120,128,144,170,176,182],{"__ignoreMap":98},[102,103,106],"span",{"class":104,"line":105},"line",1,[102,107,109],{"class":108},"s38Sx","{\n",[102,111,113,117],{"class":104,"line":112},2,[102,114,116],{"class":115},"sK71F","  \"mcpServers\"",[102,118,119],{"class":108},": {\n",[102,121,123,126],{"class":104,"line":122},3,[102,124,125],{"class":115},"    \"urls\"",[102,127,119],{"class":108},[102,129,131,134,137,141],{"class":104,"line":130},4,[102,132,133],{"class":115},"      \"command\"",[102,135,136],{"class":108},": ",[102,138,140],{"class":139},"shU9J","\"npx\"",[102,142,143],{"class":108},",\n",[102,145,147,150,153,156,159,162,164,167],{"class":104,"line":146},5,[102,148,149],{"class":115},"      \"args\"",[102,151,152],{"class":108},": [",[102,154,155],{"class":139},"\"-y\"",[102,157,158],{"class":108},", ",[102,160,161],{"class":139},"\"@agntn\u002Furls\"",[102,163,158],{"class":108},[102,165,166],{"class":139},"\"mcp\"",[102,168,169],{"class":108},"]\n",[102,171,173],{"class":104,"line":172},6,[102,174,175],{"class":108},"    }\n",[102,177,179],{"class":104,"line":178},7,[102,180,181],{"class":108},"  }\n",[102,183,185],{"class":104,"line":184},8,[102,186,187],{"class":108},"}\n",[74,189,190,191,194,195,198,199,202,203,206,207,210],{},"Set ",[78,192,193],{},"VIRUSTOTAL_API_KEY"," or ",[78,196,197],{},"URLSCAN_API_KEY"," in the server's environment when you want those sources at full strength. Building your own server? ",[78,200,201],{},"createMcpServer"," from ",[78,204,205],{},"@agntn\u002Furls\u002Fmcp"," returns a configured ",[78,208,209],{},"McpServer",".",[87,212,214],{"id":213},"ai-sdk","AI SDK",[92,216,220],{"className":217,"code":218,"language":219,"meta":98,"style":98},"language-ts shiki shiki-themes urls urls urls","import { discoverTool, providersTool } from \"@agntn\u002Furls\u002Fai\";\nimport { generateText } from \"ai\";\n\nconst { text } = await generateText({\n  model,\n  tools: { urls_discover: discoverTool, urls_providers: providersTool },\n  prompt: \"Which JavaScript files has example.com shipped over the years?\",\n});\n","ts",[78,221,222,240,254,260,280,285,290,300],{"__ignoreMap":98},[102,223,224,228,231,234,237],{"class":104,"line":105},[102,225,227],{"class":226},"skH_V","import",[102,229,230],{"class":108}," { discoverTool, providersTool } ",[102,232,233],{"class":226},"from",[102,235,236],{"class":139}," \"@agntn\u002Furls\u002Fai\"",[102,238,239],{"class":108},";\n",[102,241,242,244,247,249,252],{"class":104,"line":112},[102,243,227],{"class":226},[102,245,246],{"class":108}," { generateText } ",[102,248,233],{"class":226},[102,250,251],{"class":139}," \"ai\"",[102,253,239],{"class":108},[102,255,256],{"class":104,"line":122},[102,257,259],{"emptyLinePlaceholder":258},true,"\n",[102,261,262,265,268,271,274,277],{"class":104,"line":130},[102,263,264],{"class":226},"const",[102,266,267],{"class":108}," { text } ",[102,269,270],{"class":226},"=",[102,272,273],{"class":226}," await",[102,275,276],{"class":115}," generateText",[102,278,279],{"class":108},"({\n",[102,281,282],{"class":104,"line":146},[102,283,284],{"class":108},"  model,\n",[102,286,287],{"class":104,"line":172},[102,288,289],{"class":108},"  tools: { urls_discover: discoverTool, urls_providers: providersTool },\n",[102,291,292,295,298],{"class":104,"line":178},[102,293,294],{"class":108},"  prompt: ",[102,296,297],{"class":139},"\"Which JavaScript files has example.com shipped over the years?\"",[102,299,143],{"class":108},[102,301,302],{"class":104,"line":184},[102,303,304],{"class":108},"});\n",[74,306,307,308,311],{},"AI SDK 7 or newer. The tool passes its ",[78,309,310],{},"abortSignal"," down to the sources.",[87,313,315],{"id":314},"pi-and-omp","Pi and OMP",[92,317,321],{"className":318,"code":319,"language":320,"meta":98,"style":98},"language-bash shiki shiki-themes urls urls urls","pi install npm:@agntn\u002Furls\nomp plugin install @agntn\u002Furls\n","bash",[78,322,323,334],{"__ignoreMap":98},[102,324,325,328,331],{"class":104,"line":105},[102,326,327],{"class":115},"pi",[102,329,330],{"class":139}," install",[102,332,333],{"class":139}," npm:@agntn\u002Furls\n",[102,335,336,339,342,344],{"class":104,"line":112},[102,337,338],{"class":115},"omp",[102,340,341],{"class":139}," plugin",[102,343,330],{"class":139},[102,345,346],{"class":139}," @agntn\u002Furls\n",[74,348,349],{},"Both extensions print URLs one per line, with a note at the end when the source had more. They take the host's cancel signal too.",[87,351,353],{"id":352},"why-a-page-and-not-everything","Why a page and not everything?",[74,355,356,357,360],{},"A busy domain has tens of thousands of URLs, and an agent that gets all of them in its context isn't smarter, just slower and more expensive. So the agent tools answer with a page: at most 100 URLs per source unless the call sets ",[78,358,359],{},"limit",". The library and CLI stay unbounded.",[74,362,363],{},"Each page says what cut it:",[365,366,367,380],"table",{},[368,369,370],"thead",{},[371,372,373,377],"tr",{},[374,375,376],"th",{},"Field",[374,378,379],{},"Meaning",[381,382,383,394,403,413],"tbody",{},[371,384,385,391],{},[386,387,388],"td",{},[78,389,390],{},"count",[386,392,393],{},"URLs on this page",[371,395,396,400],{},[386,397,398],{},[78,399,359],{},[386,401,402],{},"the bound that applied, the caller's or 100",[371,404,405,410],{},[386,406,407],{},[78,408,409],{},"hasMore",[386,411,412],{},"the source had more than fits, or stopped on its own with more advertised",[371,414,415,420],{},[386,416,417],{},[78,418,419],{},"truncated",[386,421,422],{},"the source stopped on its own, its page safeguard or a cursor it won't follow",[74,424,425,427,428,430,431,433,434,158,437,194,440,443,444,446],{},[78,426,409],{}," without ",[78,429,419],{}," means the limit cut it: raise ",[78,432,359],{},", or narrow with ",[78,435,436],{},"match",[78,438,439],{},"ext",[78,441,442],{},"urlScope",". With ",[78,445,419],{}," a higher limit won't help, since the source itself gave up. Filters won't either, because they work on what the source returned. Another source might.",[448,449,451],"h3",{"id":450},"what-an-answer-looks-like","What an answer looks like",[92,453,455],{"className":94,"code":454,"language":97,"meta":98,"style":98},"{\n  \"provider\": \"wayback\",\n  \"count\": 2,\n  \"limit\": 2,\n  \"hasMore\": true,\n  \"truncated\": false,\n  \"urls\": [\n    {\n      \"url\": \"http:\u002F\u002Fexample.com:80\u002F\",\n      \"source\": \"wayback\",\n      \"input\": \"example.com\",\n      \"firstSeen\": \"2002-01-20T14:25:10Z\",\n      \"lastSeen\": \"2002-01-20T14:25:10Z\"\n    }\n  ]\n}\n",[78,456,457,461,473,481,488,496,504,512,517,530,542,555,568,579,584,590],{"__ignoreMap":98},[102,458,459],{"class":104,"line":105},[102,460,109],{"class":108},[102,462,463,466,468,471],{"class":104,"line":112},[102,464,465],{"class":115},"  \"provider\"",[102,467,136],{"class":108},[102,469,470],{"class":139},"\"wayback\"",[102,472,143],{"class":108},[102,474,475,478],{"class":104,"line":122},[102,476,477],{"class":115},"  \"count\"",[102,479,480],{"class":108},": 2,\n",[102,482,483,486],{"class":104,"line":130},[102,484,485],{"class":115},"  \"limit\"",[102,487,480],{"class":108},[102,489,490,493],{"class":104,"line":146},[102,491,492],{"class":115},"  \"hasMore\"",[102,494,495],{"class":108},": true,\n",[102,497,498,501],{"class":104,"line":172},[102,499,500],{"class":115},"  \"truncated\"",[102,502,503],{"class":108},": false,\n",[102,505,506,509],{"class":104,"line":178},[102,507,508],{"class":115},"  \"urls\"",[102,510,511],{"class":108},": [\n",[102,513,514],{"class":104,"line":184},[102,515,516],{"class":108},"    {\n",[102,518,520,523,525,528],{"class":104,"line":519},9,[102,521,522],{"class":115},"      \"url\"",[102,524,136],{"class":108},[102,526,527],{"class":139},"\"http:\u002F\u002Fexample.com:80\u002F\"",[102,529,143],{"class":108},[102,531,533,536,538,540],{"class":104,"line":532},10,[102,534,535],{"class":115},"      \"source\"",[102,537,136],{"class":108},[102,539,470],{"class":139},[102,541,143],{"class":108},[102,543,545,548,550,553],{"class":104,"line":544},11,[102,546,547],{"class":115},"      \"input\"",[102,549,136],{"class":108},[102,551,552],{"class":139},"\"example.com\"",[102,554,143],{"class":108},[102,556,558,561,563,566],{"class":104,"line":557},12,[102,559,560],{"class":115},"      \"firstSeen\"",[102,562,136],{"class":108},[102,564,565],{"class":139},"\"2002-01-20T14:25:10Z\"",[102,567,143],{"class":108},[102,569,571,574,576],{"class":104,"line":570},13,[102,572,573],{"class":115},"      \"lastSeen\"",[102,575,136],{"class":108},[102,577,578],{"class":139},"\"2002-01-20T14:25:10Z\"\n",[102,580,582],{"class":104,"line":581},14,[102,583,175],{"class":108},[102,585,587],{"class":104,"line":586},15,[102,588,589],{"class":108},"  ]\n",[102,591,593],{"class":104,"line":592},16,[102,594,187],{"class":108},[74,596,597,598,601,602,194,605,608],{},"(Shortened to one record.) With ",[78,599,600],{},"provider: \"all\""," the answer is a list of ",[78,603,604],{},"{ provider, result }",[78,606,607],{},"{ provider, error }",", one per source.",[74,610,611,612,615,616,619],{},"The query URL behind each record, ",[78,613,614],{},"reference",", stays out of the MCP and AI SDK answers. It's the same for every URL of one request and would only burn tokens. Pass ",[78,617,618],{},"reference: true"," when you need it.",[87,621,623],{"id":622},"try-it-without-an-agent","Try it without an agent",[74,625,626,627,632,633,635],{},"The ",[628,629,631],"a",{"href":630},"\u002Fdiscover","explorer"," runs ",[78,634,80],{}," through the docs worker and shows the exact text an MCP client gets.",[637,638,639],"style",{},"html pre.shiki code .s38Sx, html code.shiki .s38Sx{--shiki-light:var(--ui-text-highlighted);--shiki-default:var(--ui-text-highlighted);--shiki-dark:var(--ui-text-highlighted)}html pre.shiki code .sK71F, html code.shiki .sK71F{--shiki-light:var(--shiki-token-function);--shiki-default:var(--shiki-token-function);--shiki-dark:var(--shiki-token-function)}html pre.shiki code .shU9J, html code.shiki .shU9J{--shiki-light:var(--shiki-token-string);--shiki-default:var(--shiki-token-string);--shiki-dark:var(--shiki-token-string)}html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .skH_V, html code.shiki .skH_V{--shiki-light:var(--shiki-token-keyword);--shiki-default:var(--shiki-token-keyword);--shiki-dark:var(--shiki-token-keyword)}",{"title":98,"searchDepth":112,"depth":112,"links":641},[642,643,644,645,648],{"id":89,"depth":112,"text":90},{"id":213,"depth":112,"text":214},{"id":314,"depth":112,"text":315},{"id":352,"depth":112,"text":353,"children":646},[647],{"id":450,"depth":122,"text":451},{"id":622,"depth":112,"text":623},"Two tools on MCP and AI SDK and Pi and OMP. A bounded page per source with hasMore and truncated so the agent narrows the call instead of drowning.","md",null,{"icon":653},"i-lucide-bot",{"title":655,"description":649},"URL discovery tools for MCP and AI SDK and Pi and OMP","6S9lfZz2WAjFVGYdEDAydZD6WqkK7BXrQ6WDbQ1ZquM",[658,660],{"title":20,"path":21,"stem":22,"description":659,"children":-1},"The urls command. Discover from one source or all of them. Print bare URLs or JSONL. List the sources or start the MCP server.",{"title":28,"path":29,"stem":30,"description":661,"children":-1},"Plug your own passive index into the registry. Subclass Provider and reuse UrlCollector and register it. It shows up on every surface.",1790685300951]