[{"data":1,"prerenderedAt":183},["ShallowReactive",2],{"navigation_docs":3,"-providers-arquivo":68,"-providers-arquivo-surround":178},[4,32],{"title":5,"path":6,"stem":7,"children":8,"icon":31},"Guide","\u002Fguide","1.guide\u002F1.index",[9,11,15,19,23,27],{"title":10,"path":6,"stem":7},"Getting Started",{"title":12,"path":13,"stem":14},"Discovering URLs","\u002Fguide\u002Fdiscover","1.guide\u002F2.discover",{"title":16,"path":17,"stem":18},"Filters and scope","\u002Fguide\u002Ffilters","1.guide\u002F3.filters",{"title":20,"path":21,"stem":22},"CLI","\u002Fguide\u002Fcli","1.guide\u002F4.cli",{"title":24,"path":25,"stem":26},"Agents","\u002Fguide\u002Fagents","1.guide\u002F5.agents",{"title":28,"path":29,"stem":30},"Custom providers","\u002Fguide\u002Fcustom","1.guide\u002F6.custom","i-lucide-book-open",{"title":33,"path":34,"stem":35,"children":36,"icon":67},"Providers","\u002Fproviders","2.providers\u002F0.index",[37,39,43,47,51,55,59,63],{"title":38,"path":34,"stem":35},"Overview",{"title":40,"path":41,"stem":42},"AlienVault OTX","\u002Fproviders\u002Falienvault","2.providers\u002F01.alienvault",{"title":44,"path":45,"stem":46},"Arquivo.pt","\u002Fproviders\u002Farquivo","2.providers\u002F02.arquivo",{"title":48,"path":49,"stem":50},"Common Crawl","\u002Fproviders\u002Fcommoncrawl","2.providers\u002F03.commoncrawl",{"title":52,"path":53,"stem":54},"URLScan","\u002Fproviders\u002Furlscan","2.providers\u002F04.urlscan",{"title":56,"path":57,"stem":58},"Vefsafn","\u002Fproviders\u002Fvefsafn","2.providers\u002F05.vefsafn",{"title":60,"path":61,"stem":62},"VirusTotal","\u002Fproviders\u002Fvirustotal","2.providers\u002F06.virustotal",{"title":64,"path":65,"stem":66},"Wayback Machine","\u002Fproviders\u002Fwayback","2.providers\u002F07.wayback","i-lucide-library",{"id":69,"title":44,"body":70,"description":169,"extension":170,"links":171,"meta":172,"navigation":174,"path":45,"seo":175,"stem":46,"__hash__":177},"docs\u002F2.providers\u002F02.arquivo.md",{"type":71,"value":72,"toc":162},"minimark",[73,77,82,109,113,130,134],[74,75],"provider-facts",{"name":76},"arquivo",[78,79,81],"h2",{"id":80},"what-it-knows","What it knows",[83,84,85,86,90,91,94,95,98,99,98,102,90,105,108],"p",{},"Arquivo.pt archives the Portuguese web first, but don't let the name fool you: ",[87,88,89],"code",{},"nuxt.com"," and ",[87,92,93],{},"example.com"," both come back with captures. Every row carries a timestamp, so ",[87,96,97],{},"firstSeen",", ",[87,100,101],{},"lastSeen",[87,103,104],{},"from",[87,106,107],{},"to"," all work.",[78,110,112],{"id":111},"how-it-asks","How it asks",[83,114,115,116,119,120,98,123,90,126,129],{},"One request to ",[87,117,118],{},"\u002Fwayback\u002Fcdx"," with ",[87,121,122],{},"matchType=domain",[87,124,125],{},"output=json",[87,127,128],{},"fields=url,timestamp",", so subdomains come along. The answer is NDJSON, one capture per line, streamed and parsed as it arrives.",[78,131,133],{"id":132},"traps","Traps",[135,136,137,153,159],"ul",{},[138,139,140,141,144,145,148,149,152],"li",{},"Without a ",[87,142,143],{},"limit"," the stream answers 200 with an empty body and hangs, so the provider always sends ",[87,146,147],{},"limit=10000",". An answer that fills all 10 000 rows may have been cut by the archive, and the page says so with ",[87,150,151],{},"truncated",".",[138,154,155,158],{},[87,156,157],{},"output=txt"," is rejected, which is why this one speaks NDJSON while Wayback speaks text.",[138,160,161],{},"10 000 rows are captures, not URLs. A popular page captured a thousand times eats a thousand rows and gives you one URL.",{"title":163,"searchDepth":164,"depth":164,"links":165},"",2,[166,167,168],{"id":80,"depth":164,"text":81},{"id":111,"depth":164,"text":112},{"id":132,"depth":164,"text":133},"The CDX index of the Portuguese web archive. No key. One NDJSON request with dates and a hard cap of 10 000 rows.","md",null,{"icon":173},"i-lucide-landmark",true,{"title":176,"description":169},"Arquivo.pt CDX URL discovery with capture dates","-dNAzHy2D0Xrvpbvs0Fhc0teU7V5IqlD7ri_p1clM3o",[179,181],{"title":40,"path":41,"stem":42,"description":180,"children":-1},"The URL list of the Open Threat Exchange. No key. Paged JSON up to 20 pages. Rate limits under parallel use.",{"title":48,"path":49,"stem":50,"description":182,"children":-1},"The crawl indexes of Common Crawl. No key. The newest index of each of the last five years queried one by one.",1790685300952]