[{"data":1,"prerenderedAt":799},["ShallowReactive",2],{"navigation_docs":3,"-guide-discover":68,"-guide-discover-surround":794},[4,32],{"title":5,"path":6,"stem":7,"children":8,"icon":31},"Guide","\u002Fguide","1.guide\u002F1.index",[9,11,15,19,23,27],{"title":10,"path":6,"stem":7},"Getting Started",{"title":12,"path":13,"stem":14},"Discovering URLs","\u002Fguide\u002Fdiscover","1.guide\u002F2.discover",{"title":16,"path":17,"stem":18},"Filters and scope","\u002Fguide\u002Ffilters","1.guide\u002F3.filters",{"title":20,"path":21,"stem":22},"CLI","\u002Fguide\u002Fcli","1.guide\u002F4.cli",{"title":24,"path":25,"stem":26},"Agents","\u002Fguide\u002Fagents","1.guide\u002F5.agents",{"title":28,"path":29,"stem":30},"Custom providers","\u002Fguide\u002Fcustom","1.guide\u002F6.custom","i-lucide-book-open",{"title":33,"path":34,"stem":35,"children":36,"icon":67},"Providers","\u002Fproviders","2.providers\u002F0.index",[37,39,43,47,51,55,59,63],{"title":38,"path":34,"stem":35},"Overview",{"title":40,"path":41,"stem":42},"AlienVault OTX","\u002Fproviders\u002Falienvault","2.providers\u002F01.alienvault",{"title":44,"path":45,"stem":46},"Arquivo.pt","\u002Fproviders\u002Farquivo","2.providers\u002F02.arquivo",{"title":48,"path":49,"stem":50},"Common Crawl","\u002Fproviders\u002Fcommoncrawl","2.providers\u002F03.commoncrawl",{"title":52,"path":53,"stem":54},"URLScan","\u002Fproviders\u002Furlscan","2.providers\u002F04.urlscan",{"title":56,"path":57,"stem":58},"Vefsafn","\u002Fproviders\u002Fvefsafn","2.providers\u002F05.vefsafn",{"title":60,"path":61,"stem":62},"VirusTotal","\u002Fproviders\u002Fvirustotal","2.providers\u002F06.virustotal",{"title":64,"path":65,"stem":66},"Wayback Machine","\u002Fproviders\u002Fwayback","2.providers\u002F07.wayback","i-lucide-library",{"id":69,"title":12,"body":70,"description":786,"extension":787,"links":788,"meta":789,"navigation":120,"path":13,"seo":791,"stem":14,"__hash__":793},"docs\u002F1.guide\u002F2.discover.md",{"type":71,"value":72,"toc":772},"minimark",[73,77,82,176,195,212,238,249,253,296,301,308,312,315,318,322,425,428,431,435,575,582,596,600,633,637,644,649,653,722,725,729,758,768],[74,75,76],"p",{},"Three ways to ask. They differ in who picks the source and in what a failure does.",[78,79,81],"h2",{"id":80},"ask-one-source-by-name","Ask one source by name",[83,84,89],"pre",{"className":85,"code":86,"language":87,"meta":88,"style":88},"language-ts shiki shiki-themes urls urls urls","import { create } from \"@agntn\u002Furls\";\n\nconst urlscan = await create(\"urlscan\");\nconst hits = await urlscan.discover(\"example.com\", { limit: 100 });\n","ts","",[90,91,92,115,122,150],"code",{"__ignoreMap":88},[93,94,97,101,105,108,112],"span",{"class":95,"line":96},"line",1,[93,98,100],{"class":99},"skH_V","import",[93,102,104],{"class":103},"s38Sx"," { create } ",[93,106,107],{"class":99},"from",[93,109,111],{"class":110},"shU9J"," \"@agntn\u002Furls\"",[93,113,114],{"class":103},";\n",[93,116,118],{"class":95,"line":117},2,[93,119,121],{"emptyLinePlaceholder":120},true,"\n",[93,123,125,128,131,134,137,141,144,147],{"class":95,"line":124},3,[93,126,127],{"class":99},"const",[93,129,130],{"class":103}," urlscan ",[93,132,133],{"class":99},"=",[93,135,136],{"class":99}," await",[93,138,140],{"class":139},"sK71F"," create",[93,142,143],{"class":103},"(",[93,145,146],{"class":110},"\"urlscan\"",[93,148,149],{"class":103},");\n",[93,151,153,155,158,160,162,165,168,170,173],{"class":95,"line":152},4,[93,154,127],{"class":99},[93,156,157],{"class":103}," hits ",[93,159,133],{"class":99},[93,161,136],{"class":99},[93,163,164],{"class":103}," urlscan.",[93,166,167],{"class":139},"discover",[93,169,143],{"class":103},[93,171,172],{"class":110},"\"example.com\"",[93,174,175],{"class":103},", { limit: 100 });\n",[74,177,178,179,182,183,186,187,190,191,194],{},"The registry key is the name you pass everywhere: ",[90,180,181],{},"create()",", the CLI's ",[90,184,185],{},"-p",", the ",[90,188,189],{},"provider"," argument of every agent tool. ",[192,193,33],"a",{"href":34}," lists all of them.",[74,196,197,199,200,203,204,207,208,211],{},[90,198,181],{}," also takes a config object: ",[90,201,202],{},"apiKey",", ",[90,205,206],{},"baseUrl"," for your own mirror of the endpoint, and ",[90,209,210],{},"timeout"," in milliseconds.",[83,213,215],{"className":85,"code":214,"language":87,"meta":88,"style":88},"const virustotal = await create(\"virustotal\", { apiKey: process.env.VT_KEY, timeout: 20_000 });\n",[90,216,217],{"__ignoreMap":88},[93,218,219,221,224,226,228,230,232,235],{"class":95,"line":96},[93,220,127],{"class":99},[93,222,223],{"class":103}," virustotal ",[93,225,133],{"class":99},[93,227,136],{"class":99},[93,229,140],{"class":139},[93,231,143],{"class":103},[93,233,234],{"class":110},"\"virustotal\"",[93,236,237],{"class":103},", { apiKey: process.env.VT_KEY, timeout: 20_000 });\n",[74,239,240,241,244,245,248],{},"Prefer a direct import? Each source is its own subpath: ",[90,242,243],{},"@agntn\u002Furls\u002Fproviders\u002Fwayback"," exports ",[90,246,247],{},"Wayback",".",[78,250,252],{"id":251},"let-the-library-pick","Let the library pick",[83,254,256],{"className":85,"code":255,"language":87,"meta":88,"style":88},"import { discoverWithFallback } from \"@agntn\u002Furls\";\n\nconst { provider, result } = await discoverWithFallback(\"example.com\", { limit: 50 });\n",[90,257,258,271,275],{"__ignoreMap":88},[93,259,260,262,265,267,269],{"class":95,"line":96},[93,261,100],{"class":99},[93,263,264],{"class":103}," { discoverWithFallback } ",[93,266,107],{"class":99},[93,268,111],{"class":110},[93,270,114],{"class":103},[93,272,273],{"class":95,"line":117},[93,274,121],{"emptyLinePlaceholder":120},[93,276,277,279,282,284,286,289,291,293],{"class":95,"line":124},[93,278,127],{"class":99},[93,280,281],{"class":103}," { provider, result } ",[93,283,133],{"class":99},[93,285,136],{"class":99},[93,287,288],{"class":139}," discoverWithFallback",[93,290,143],{"class":103},[93,292,172],{"class":110},[93,294,295],{"class":103},", { limit: 50 });\n",[297,298,300],"h3",{"id":299},"which-source-goes-first","Which source goes first?",[74,302,303,304,307],{},"VirusTotal goes first when ",[90,305,306],{},"VIRUSTOTAL_API_KEY"," is set. Without it the first try is AlienVault OTX, the keyless default.",[297,309,311],{"id":310},"when-does-it-move-on","When does it move on?",[74,313,314],{},"Only when the failure is about access: a missing or rejected key, a payment wall, a rate limit, an operation the source doesn't serve. Then it tries the next source in registry order.",[74,316,317],{},"A network error or a bad domain doesn't fall through. It's thrown right away, because retrying the same broken request on every other source just hides the problem.",[78,319,321],{"id":320},"ask-every-source-at-once","Ask every source at once",[83,323,325],{"className":85,"code":324,"language":87,"meta":88,"style":88},"import { discoverAll } from \"@agntn\u002Furls\";\n\nconst outcomes = await discoverAll(\"example.com\", { limit: 20 });\n\nfor (const outcome of outcomes) {\n  if (outcome.error) console.warn(outcome.provider, outcome.error.message);\n  else console.log(outcome.provider, outcome.result.length);\n}\n",[90,326,327,340,344,365,369,389,404,419],{"__ignoreMap":88},[93,328,329,331,334,336,338],{"class":95,"line":96},[93,330,100],{"class":99},[93,332,333],{"class":103}," { discoverAll } ",[93,335,107],{"class":99},[93,337,111],{"class":110},[93,339,114],{"class":103},[93,341,342],{"class":95,"line":117},[93,343,121],{"emptyLinePlaceholder":120},[93,345,346,348,351,353,355,358,360,362],{"class":95,"line":124},[93,347,127],{"class":99},[93,349,350],{"class":103}," outcomes ",[93,352,133],{"class":99},[93,354,136],{"class":99},[93,356,357],{"class":139}," discoverAll",[93,359,143],{"class":103},[93,361,172],{"class":110},[93,363,364],{"class":103},", { limit: 20 });\n",[93,366,367],{"class":95,"line":152},[93,368,121],{"emptyLinePlaceholder":120},[93,370,372,375,378,380,383,386],{"class":95,"line":371},5,[93,373,374],{"class":99},"for",[93,376,377],{"class":103}," (",[93,379,127],{"class":99},[93,381,382],{"class":103}," outcome ",[93,384,385],{"class":99},"of",[93,387,388],{"class":103}," outcomes) {\n",[93,390,392,395,398,401],{"class":95,"line":391},6,[93,393,394],{"class":99},"  if",[93,396,397],{"class":103}," (outcome.error) console.",[93,399,400],{"class":139},"warn",[93,402,403],{"class":103},"(outcome.provider, outcome.error.message);\n",[93,405,407,410,413,416],{"class":95,"line":406},7,[93,408,409],{"class":99},"  else",[93,411,412],{"class":103}," console.",[93,414,415],{"class":139},"log",[93,417,418],{"class":103},"(outcome.provider, outcome.result.length);\n",[93,420,422],{"class":95,"line":421},8,[93,423,424],{"class":103},"}\n",[74,426,427],{},"Every source runs in parallel and reports on its own. Everybody talks at once and nobody waits for the slowpoke: one source timing out doesn't take the others down with it, and you see exactly who failed and why.",[74,429,430],{},"The limit applies per source, not to the whole comparison. Deduplication is per source too, so the same URL from Wayback and from Arquivo.pt shows up twice, once under each name. That's on purpose: two sources agreeing on a URL is information.",[78,432,434],{"id":433},"what-a-record-looks-like","What a record looks like",[83,436,438],{"className":85,"code":437,"language":87,"meta":88,"style":88},"interface DiscoveredUrl {\n  url: string; \u002F\u002F as the source wrote it, first spelling wins\n  source: string; \u002F\u002F registry key of the source\n  input: string; \u002F\u002F the domain you asked about\n  reference?: string; \u002F\u002F the query URL that returned it\n  ext?: string; \u002F\u002F path extension, no dot\n  queryKeys?: string[]; \u002F\u002F query keys left after tracking keys are dropped\n  firstSeen?: string; \u002F\u002F earliest capture the walk read, ISO 8601\n  lastSeen?: string; \u002F\u002F latest capture the walk read, ISO 8601\n}\n",[90,439,440,451,469,483,497,512,526,541,555,570],{"__ignoreMap":88},[93,441,442,445,448],{"class":95,"line":96},[93,443,444],{"class":99},"interface",[93,446,447],{"class":139}," DiscoveredUrl",[93,449,450],{"class":103}," {\n",[93,452,453,456,459,462,465],{"class":95,"line":117},[93,454,455],{"class":103},"  url",[93,457,458],{"class":99},":",[93,460,461],{"class":139}," string",[93,463,464],{"class":103},"; ",[93,466,468],{"class":467},"scIB-","\u002F\u002F as the source wrote it, first spelling wins\n",[93,470,471,474,476,478,480],{"class":95,"line":124},[93,472,473],{"class":103},"  source",[93,475,458],{"class":99},[93,477,461],{"class":139},[93,479,464],{"class":103},[93,481,482],{"class":467},"\u002F\u002F registry key of the source\n",[93,484,485,488,490,492,494],{"class":95,"line":152},[93,486,487],{"class":103},"  input",[93,489,458],{"class":99},[93,491,461],{"class":139},[93,493,464],{"class":103},[93,495,496],{"class":467},"\u002F\u002F the domain you asked about\n",[93,498,499,502,505,507,509],{"class":95,"line":371},[93,500,501],{"class":103},"  reference",[93,503,504],{"class":99},"?:",[93,506,461],{"class":139},[93,508,464],{"class":103},[93,510,511],{"class":467},"\u002F\u002F the query URL that returned it\n",[93,513,514,517,519,521,523],{"class":95,"line":391},[93,515,516],{"class":103},"  ext",[93,518,504],{"class":99},[93,520,461],{"class":139},[93,522,464],{"class":103},[93,524,525],{"class":467},"\u002F\u002F path extension, no dot\n",[93,527,528,531,533,535,538],{"class":95,"line":406},[93,529,530],{"class":103},"  queryKeys",[93,532,504],{"class":99},[93,534,461],{"class":139},[93,536,537],{"class":103},"[]; ",[93,539,540],{"class":467},"\u002F\u002F query keys left after tracking keys are dropped\n",[93,542,543,546,548,550,552],{"class":95,"line":421},[93,544,545],{"class":103},"  firstSeen",[93,547,504],{"class":99},[93,549,461],{"class":139},[93,551,464],{"class":103},[93,553,554],{"class":467},"\u002F\u002F earliest capture the walk read, ISO 8601\n",[93,556,558,561,563,565,567],{"class":95,"line":557},9,[93,559,560],{"class":103},"  lastSeen",[93,562,504],{"class":99},[93,564,461],{"class":139},[93,566,464],{"class":103},[93,568,569],{"class":467},"\u002F\u002F latest capture the walk read, ISO 8601\n",[93,571,573],{"class":95,"line":572},10,[93,574,424],{"class":103},[74,576,577,578,581],{},"Only the archives with a CDX index give dates: Wayback, Arquivo.pt, Vefsafn and Common Crawl. AlienVault OTX, URLScan and VirusTotal leave ",[90,579,580],{},"firstSeen"," empty.",[74,583,584,585,588,589,591,592,595],{},"A walk that ",[90,586,587],{},"limit"," ends early hasn't read the captures after the cut, so ",[90,590,580],{}," and ",[90,593,594],{},"lastSeen"," describe what it read, not the source's whole memory of that URL.",[297,597,599],{"id":598},"how-deduplication-decides-two-urls-are-the-same","How deduplication decides two URLs are the same",[74,601,602,605,606,591,609,612,613,591,616,619,620,591,623,626,627,591,630,248],{},[90,603,604],{},"normalizeUrl"," lowercases scheme and host, drops the fragment, sorts the query, strips tracking keys like ",[90,607,608],{},"utm_source",[90,610,611],{},"fbclid",", and trims a trailing slash. ",[90,614,615],{},"http:\u002F\u002Fexample.com:80\u002F",[90,617,618],{},"http:\u002F\u002Fexample.com\u002F"," are one URL. ",[90,621,622],{},"http:\u002F\u002F",[90,624,625],{},"https:\u002F\u002F"," are two, and so are ",[90,628,629],{},"example.com",[90,631,632],{},"www.example.com",[78,634,636],{"id":635},"how-much-comes-back","How much comes back",[74,638,639,640,643],{},"Without a limit the library and CLI run a source to the end, up to its own page safeguard. AlienVault OTX stops after 20 pages, URLScan and VirusTotal after 50. The published ceiling is ",[90,641,642],{},"MAX_DISCOVER_RESULTS",", 100 000 URLs, and a bigger limit is clamped to it.",[74,645,646,647,248],{},"The agent tools are stricter on purpose. See ",[192,648,24],{"href":25},[78,650,652],{"id":651},"cancelling","Cancelling",[83,654,656],{"className":85,"code":655,"language":87,"meta":88,"style":88},"const controller = new AbortController();\nsetTimeout(() => controller.abort(), 10_000);\n\nconst hits = await wayback.discover(\"example.com\", { signal: controller.signal });\n",[90,657,658,676,696,700],{"__ignoreMap":88},[93,659,660,662,665,667,670,673],{"class":95,"line":96},[93,661,127],{"class":99},[93,663,664],{"class":103}," controller ",[93,666,133],{"class":99},[93,668,669],{"class":99}," new",[93,671,672],{"class":139}," AbortController",[93,674,675],{"class":103},"();\n",[93,677,678,681,684,687,690,693],{"class":95,"line":117},[93,679,680],{"class":139},"setTimeout",[93,682,683],{"class":103},"(() ",[93,685,686],{"class":99},"=>",[93,688,689],{"class":103}," controller.",[93,691,692],{"class":139},"abort",[93,694,695],{"class":103},"(), 10_000);\n",[93,697,698],{"class":95,"line":124},[93,699,121],{"emptyLinePlaceholder":120},[93,701,702,704,706,708,710,713,715,717,719],{"class":95,"line":152},[93,703,127],{"class":99},[93,705,157],{"class":103},[93,707,133],{"class":99},[93,709,136],{"class":99},[93,711,712],{"class":103}," wayback.",[93,714,167],{"class":139},[93,716,143],{"class":103},[93,718,172],{"class":110},[93,720,721],{"class":103},", { signal: controller.signal });\n",[74,723,724],{},"The signal reaches every request a source makes. An abort surfaces as your own reason, not as a network error.",[78,726,728],{"id":727},"when-something-fails","When something fails",[74,730,731,732,735,736,203,739,203,742,203,745,203,748,203,751,203,754,757],{},"Failures are ",[90,733,734],{},"UrlsError"," or one of its subclasses: ",[90,737,738],{},"AuthError",[90,740,741],{},"RateLimitError",[90,743,744],{},"PaymentError",[90,746,747],{},"HTTPError",[90,749,750],{},"NotFoundError",[90,752,753],{},"InvalidInputError",[90,755,756],{},"UnknownProviderError",". Query strings that look like secrets are redacted from every message before you see it.",[74,759,760,761,763,764,767],{},"Bad input fails before any request goes out. ",[90,762,181],{}," rejects an unknown key, and ",[90,765,766],{},"discover()"," rejects a domain it can't turn into a hostname.",[769,770,771],"style",{},"html pre.shiki code .skH_V, html code.shiki .skH_V{--shiki-light:var(--shiki-token-keyword);--shiki-default:var(--shiki-token-keyword);--shiki-dark:var(--shiki-token-keyword)}html pre.shiki code .s38Sx, html code.shiki .s38Sx{--shiki-light:var(--ui-text-highlighted);--shiki-default:var(--ui-text-highlighted);--shiki-dark:var(--ui-text-highlighted)}html pre.shiki code .shU9J, html code.shiki .shU9J{--shiki-light:var(--shiki-token-string);--shiki-default:var(--shiki-token-string);--shiki-dark:var(--shiki-token-string)}html pre.shiki code .sK71F, html code.shiki .sK71F{--shiki-light:var(--shiki-token-function);--shiki-default:var(--shiki-token-function);--shiki-dark:var(--shiki-token-function)}html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html pre.shiki code .scIB-, html code.shiki .scIB-{--shiki-light:var(--shiki-token-comment);--shiki-default:var(--shiki-token-comment);--shiki-dark:var(--shiki-token-comment)}",{"title":88,"searchDepth":117,"depth":117,"links":773},[774,775,779,780,783,784,785],{"id":80,"depth":117,"text":81},{"id":251,"depth":117,"text":252,"children":776},[777,778],{"id":299,"depth":124,"text":300},{"id":310,"depth":124,"text":311},{"id":320,"depth":117,"text":321},{"id":433,"depth":117,"text":434,"children":781},[782],{"id":598,"depth":124,"text":599},{"id":635,"depth":117,"text":636},{"id":651,"depth":117,"text":652},{"id":727,"depth":117,"text":728},"One source by name. The automatic pick with fallback. Or every source side by side with its own results and errors.","md",null,{"icon":790},"i-lucide-radar",{"title":792,"description":786},"Discover URLs for a domain from one source or all of them","dz_63H0hpyfCUKLNuMpfS6_IlJLc5Aj803Muk4J78Pw",[795,797],{"title":10,"path":6,"stem":7,"description":796,"children":-1},"Install the package and ask one source what URLs it knows for a domain. Library and CLI in a few lines.",{"title":16,"path":17,"stem":18,"description":798,"children":-1},"Default host scope. URL prefixes and globs. Extensions and query keys and substrings and dates. What each filter keeps and why the limit counts only survivors.",1790685303069]