[{"data":1,"prerenderedAt":147},["ShallowReactive",2],{"navigation_docs":3,"-providers-commoncrawl":68,"-providers-commoncrawl-surround":142},[4,32],{"title":5,"path":6,"stem":7,"children":8,"icon":31},"Guide","\u002Fguide","1.guide\u002F1.index",[9,11,15,19,23,27],{"title":10,"path":6,"stem":7},"Getting Started",{"title":12,"path":13,"stem":14},"Discovering URLs","\u002Fguide\u002Fdiscover","1.guide\u002F2.discover",{"title":16,"path":17,"stem":18},"Filters and scope","\u002Fguide\u002Ffilters","1.guide\u002F3.filters",{"title":20,"path":21,"stem":22},"CLI","\u002Fguide\u002Fcli","1.guide\u002F4.cli",{"title":24,"path":25,"stem":26},"Agents","\u002Fguide\u002Fagents","1.guide\u002F5.agents",{"title":28,"path":29,"stem":30},"Custom providers","\u002Fguide\u002Fcustom","1.guide\u002F6.custom","i-lucide-book-open",{"title":33,"path":34,"stem":35,"children":36,"icon":67},"Providers","\u002Fproviders","2.providers\u002F0.index",[37,39,43,47,51,55,59,63],{"title":38,"path":34,"stem":35},"Overview",{"title":40,"path":41,"stem":42},"AlienVault OTX","\u002Fproviders\u002Falienvault","2.providers\u002F01.alienvault",{"title":44,"path":45,"stem":46},"Arquivo.pt","\u002Fproviders\u002Farquivo","2.providers\u002F02.arquivo",{"title":48,"path":49,"stem":50},"Common Crawl","\u002Fproviders\u002Fcommoncrawl","2.providers\u002F03.commoncrawl",{"title":52,"path":53,"stem":54},"URLScan","\u002Fproviders\u002Furlscan","2.providers\u002F04.urlscan",{"title":56,"path":57,"stem":58},"Vefsafn","\u002Fproviders\u002Fvefsafn","2.providers\u002F05.vefsafn",{"title":60,"path":61,"stem":62},"VirusTotal","\u002Fproviders\u002Fvirustotal","2.providers\u002F06.virustotal",{"title":64,"path":65,"stem":66},"Wayback Machine","\u002Fproviders\u002Fwayback","2.providers\u002F07.wayback","i-lucide-library",{"id":69,"title":48,"body":70,"description":133,"extension":134,"links":135,"meta":136,"navigation":138,"path":49,"seo":139,"stem":50,"__hash__":141},"docs\u002F2.providers\u002F03.commoncrawl.md",{"type":71,"value":72,"toc":126},"minimark",[73,77,82,86,90,105,109],[74,75],"provider-facts",{"name":76},"commoncrawl",[78,79,81],"h2",{"id":80},"what-it-knows","What it knows",[83,84,85],"p",{},"Common Crawl publishes a few crawls a year, each with its own CDX index. Together they cover a big slice of the public web, and every row has a timestamp.",[78,87,89],{"id":88},"how-it-asks","How it asks",[83,91,92,93,97,98,101,102,104],{},"First ",[94,95,96],"code",{},"collinfo.json",", the list of all crawl indexes. Then, for each of the last five calendar years, the newest index whose id names that year, queried for ",[94,99,100],{},"*.domain"," as CDX text. Only index URLs on the configured host are followed, so a strange entry in ",[94,103,96],{}," can't send your request somewhere else.",[78,106,108],{"id":107},"traps","Traps",[110,111,112,116,123],"ul",{},[113,114,115],"li",{},"The index frontend throws 502 and 504 now and then. One failing index is skipped and the other years still answer. A caller abort still stops everything.",[113,117,118,119,122],{},"From some networks ",[94,120,121],{},"index.commoncrawl.org"," refuses connections entirely. The provider's tests run on mocked HTTP for that reason, and a live failure there usually means the network, not the code.",[113,124,125],{},"Five years back is the whole range. Older crawls aren't asked.",{"title":127,"searchDepth":128,"depth":128,"links":129},"",2,[130,131,132],{"id":80,"depth":128,"text":81},{"id":88,"depth":128,"text":89},{"id":107,"depth":128,"text":108},"The crawl indexes of Common Crawl. No key. The newest index of each of the last five years queried one by one.","md",null,{"icon":137},"i-lucide-database",true,{"title":140,"description":133},"Common Crawl index URL discovery for a domain","L-wl1CbSraZ4iHe8l6_wsIHJtD8lFHi-q8U4_-zswPw",[143,145],{"title":44,"path":45,"stem":46,"description":144,"children":-1},"The CDX index of the Portuguese web archive. No key. One NDJSON request with dates and a hard cap of 10 000 rows.",{"title":52,"path":53,"stem":54,"description":146,"children":-1},"The search API of urlscan.io. Works without a key at small volumes. Cursor pages up to 50 and more room with URLSCAN_API_KEY.",1790685300952]