[{"data":1,"prerenderedAt":182},["ShallowReactive",2],{"navigation_docs":3,"-providers-wayback":68,"-providers-wayback-surround":179},[4,32],{"title":5,"path":6,"stem":7,"children":8,"icon":31},"Guide","\u002Fguide","1.guide\u002F1.index",[9,11,15,19,23,27],{"title":10,"path":6,"stem":7},"Getting Started",{"title":12,"path":13,"stem":14},"Discovering URLs","\u002Fguide\u002Fdiscover","1.guide\u002F2.discover",{"title":16,"path":17,"stem":18},"Filters and scope","\u002Fguide\u002Ffilters","1.guide\u002F3.filters",{"title":20,"path":21,"stem":22},"CLI","\u002Fguide\u002Fcli","1.guide\u002F4.cli",{"title":24,"path":25,"stem":26},"Agents","\u002Fguide\u002Fagents","1.guide\u002F5.agents",{"title":28,"path":29,"stem":30},"Custom providers","\u002Fguide\u002Fcustom","1.guide\u002F6.custom","i-lucide-book-open",{"title":33,"path":34,"stem":35,"children":36,"icon":67},"Providers","\u002Fproviders","2.providers\u002F0.index",[37,39,43,47,51,55,59,63],{"title":38,"path":34,"stem":35},"Overview",{"title":40,"path":41,"stem":42},"AlienVault OTX","\u002Fproviders\u002Falienvault","2.providers\u002F01.alienvault",{"title":44,"path":45,"stem":46},"Arquivo.pt","\u002Fproviders\u002Farquivo","2.providers\u002F02.arquivo",{"title":48,"path":49,"stem":50},"Common Crawl","\u002Fproviders\u002Fcommoncrawl","2.providers\u002F03.commoncrawl",{"title":52,"path":53,"stem":54},"URLScan","\u002Fproviders\u002Furlscan","2.providers\u002F04.urlscan",{"title":56,"path":57,"stem":58},"Vefsafn","\u002Fproviders\u002Fvefsafn","2.providers\u002F05.vefsafn",{"title":60,"path":61,"stem":62},"VirusTotal","\u002Fproviders\u002Fvirustotal","2.providers\u002F06.virustotal",{"title":64,"path":65,"stem":66},"Wayback Machine","\u002Fproviders\u002Fwayback","2.providers\u002F07.wayback","i-lucide-library",{"id":69,"title":64,"body":70,"description":170,"extension":171,"links":172,"meta":173,"navigation":175,"path":65,"seo":176,"stem":66,"__hash__":178},"docs\u002F2.providers\u002F07.wayback.md",{"type":71,"value":72,"toc":163},"minimark",[73,77,82,86,90,110,114],[74,75],"provider-facts",{"name":76},"wayback",[78,79,81],"h2",{"id":80},"what-it-knows","What it knows",[83,84,85],"p",{},"Everything the Internet Archive captured under the domain since 1996. For an old domain that's the deepest source by far, and every row has a timestamp.",[78,87,89],{"id":88},"how-it-asks","How it asks",[83,91,92,93,97,98,101,102,105,106,109],{},"One request to ",[94,95,96],"code",{},"\u002Fcdx\u002Fsearch\u002Fcdx"," with ",[94,99,100],{},"matchType=domain",", ",[94,103,104],{},"output=txt"," and ",[94,107,108],{},"fl=original,timestamp",". The answer streams line by line and the reader is cancelled the moment the collector has enough.",[78,111,113],{"id":112},"traps","Traps",[115,116,117,135,141,152],"ul",{},[118,119,120,123,124,127,128,130,131,134],"li",{},[94,121,122],{},"url=domain\u002F*"," only matches the apex host, ",[94,125,126],{},"www."," included. Subdomains never came back that way, so the provider asks with ",[94,129,100],{},", like urlfinder's ",[94,132,133],{},"*.domain\u002F*",".",[118,136,137,140],{},[94,138,139],{},"collapse=urlkey"," would dedupe on the server, but that variant times out from some networks while the plain query answers at once. The collector dedupes instead.",[118,142,143,144,147,148,151],{},"Rows are captures, not URLs. ",[94,145,146],{},"http:\u002F\u002Fexample.com:80\u002F"," captured in 2002 and ",[94,149,150],{},"http:\u002F\u002Fexample.com\u002F"," captured in 2020 are one URL after dedup. The first spelling wins and the dates widen.",[118,153,154,155,158,159,162],{},"On a big domain the full dump is huge. Set a ",[94,156,157],{},"limit"," or a filter, or run your own CDX server and point ",[94,160,161],{},"baseUrl"," at it.",{"title":164,"searchDepth":165,"depth":165,"links":166},"",2,[167,168,169],{"id":80,"depth":165,"text":81},{"id":88,"depth":165,"text":89},{"id":112,"depth":165,"text":113},"The CDX index of the Internet Archive. No key. One streamed text request with dates and subdomains included.","md",null,{"icon":174},"i-simple-icons-internetarchive",true,{"title":177,"description":170},"Wayback Machine CDX URL discovery for a domain","maCdQhWDVrKsDpur43GTF51grS0ezeutStVeKM9CV0k",[180,172],{"title":60,"path":61,"stem":62,"description":181,"children":-1},"The v3 domain URL relationship of VirusTotal. Needs VIRUSTOTAL_API_KEY. Cursor pages up to 50.",1790685300952]