[{"data":1,"prerenderedAt":526},["ShallowReactive",2],{"navigation_docs":3,"-guide-filters":68,"-guide-filters-surround":521},[4,32],{"title":5,"path":6,"stem":7,"children":8,"icon":31},"Guide","\u002Fguide","1.guide\u002F1.index",[9,11,15,19,23,27],{"title":10,"path":6,"stem":7},"Getting Started",{"title":12,"path":13,"stem":14},"Discovering URLs","\u002Fguide\u002Fdiscover","1.guide\u002F2.discover",{"title":16,"path":17,"stem":18},"Filters and scope","\u002Fguide\u002Ffilters","1.guide\u002F3.filters",{"title":20,"path":21,"stem":22},"CLI","\u002Fguide\u002Fcli","1.guide\u002F4.cli",{"title":24,"path":25,"stem":26},"Agents","\u002Fguide\u002Fagents","1.guide\u002F5.agents",{"title":28,"path":29,"stem":30},"Custom providers","\u002Fguide\u002Fcustom","1.guide\u002F6.custom","i-lucide-book-open",{"title":33,"path":34,"stem":35,"children":36,"icon":67},"Providers","\u002Fproviders","2.providers\u002F0.index",[37,39,43,47,51,55,59,63],{"title":38,"path":34,"stem":35},"Overview",{"title":40,"path":41,"stem":42},"AlienVault OTX","\u002Fproviders\u002Falienvault","2.providers\u002F01.alienvault",{"title":44,"path":45,"stem":46},"Arquivo.pt","\u002Fproviders\u002Farquivo","2.providers\u002F02.arquivo",{"title":48,"path":49,"stem":50},"Common Crawl","\u002Fproviders\u002Fcommoncrawl","2.providers\u002F03.commoncrawl",{"title":52,"path":53,"stem":54},"URLScan","\u002Fproviders\u002Furlscan","2.providers\u002F04.urlscan",{"title":56,"path":57,"stem":58},"Vefsafn","\u002Fproviders\u002Fvefsafn","2.providers\u002F05.vefsafn",{"title":60,"path":61,"stem":62},"VirusTotal","\u002Fproviders\u002Fvirustotal","2.providers\u002F06.virustotal",{"title":64,"path":65,"stem":66},"Wayback Machine","\u002Fproviders\u002Fwayback","2.providers\u002F07.wayback","i-lucide-library",{"id":69,"title":16,"body":70,"description":512,"extension":513,"links":514,"meta":515,"navigation":517,"path":17,"seo":518,"stem":18,"__hash__":520},"docs\u002F1.guide\u002F3.filters.md",{"type":71,"value":72,"toc":499},"minimark",[73,77,85,90,105,112,116,126,191,196,229,246,262,266,276,315,319,332,346,350,374,377,381,384,388,407,411,414,489,495],[74,75,76],"p",{},"A busy domain has tens of thousands of archived URLs, and a good share of them is noise: font files, cache busters, templates somebody forgot to render. Filters cut that down inside each source, as the URLs arrive.",[74,78,79,80,84],{},"That detail matters. The collector applies every rule before it counts, so ",[81,82,83],"code",{},"limit: 100"," means 100 URLs that passed, not 100 raw rows with 90 of them thrown away afterwards.",[86,87,89],"h2",{"id":88},"default-scope-the-domain-and-its-subdomains","Default scope: the domain and its subdomains",[74,91,92,93,96,97,100,101,104],{},"Without any option a URL stays only when its host is the domain you asked for or a subdomain of it. Ask for ",[81,94,95],{},"mozilla.org"," and ",[81,98,99],{},"developer.mozilla.org"," stays, ",[81,102,103],{},"mozilla.org.evil.example"," goes.",[74,106,107,108,111],{},"Sources don't always stay on topic. A search index can hand back a page on another host that merely touched yours, and the host check is the safety net. ",[81,109,110],{},"noScope: true"," turns it off when you want everything a source has.",[86,113,115],{"id":114},"url-scope-prefixes-and-globs","URL scope: prefixes and globs",[74,117,118,121,122,125],{},[81,119,120],{},"urlScope"," keeps URLs that match at least one pattern. ",[81,123,124],{},"urlOutScope"," drops URLs that match any. Both look at the whole URL, scheme included, not at the hostname.",[127,128,133],"pre",{"className":129,"code":130,"language":131,"meta":132,"style":132},"language-ts shiki shiki-themes urls urls urls","await wayback.discover(\"nuxt.com\", {\n  urlScope: [\"https:\u002F\u002Fnuxt.com\u002Fdocs\"],\n  urlOutScope: [\"*\u002F_fonts\u002F*\"],\n});\n","ts","",[81,134,135,162,174,185],{"__ignoreMap":132},[136,137,140,144,148,152,155,159],"span",{"class":138,"line":139},"line",1,[136,141,143],{"class":142},"skH_V","await",[136,145,147],{"class":146},"s38Sx"," wayback.",[136,149,151],{"class":150},"sK71F","discover",[136,153,154],{"class":146},"(",[136,156,158],{"class":157},"shU9J","\"nuxt.com\"",[136,160,161],{"class":146},", {\n",[136,163,165,168,171],{"class":138,"line":164},2,[136,166,167],{"class":146},"  urlScope: [",[136,169,170],{"class":157},"\"https:\u002F\u002Fnuxt.com\u002Fdocs\"",[136,172,173],{"class":146},"],\n",[136,175,177,180,183],{"class":138,"line":176},3,[136,178,179],{"class":146},"  urlOutScope: [",[136,181,182],{"class":157},"\"*\u002F_fonts\u002F*\"",[136,184,173],{"class":146},[136,186,188],{"class":138,"line":187},4,[136,189,190],{"class":146},"});\n",[192,193,195],"h3",{"id":194},"is-a-pattern-a-prefix-or-a-glob","Is a pattern a prefix or a glob?",[74,197,198,199,202,203,206,207,96,209,212,213,216,217,220,221,224,225,228],{},"Without ",[81,200,201],{},"*"," it's a prefix with a boundary. ",[81,204,205],{},"https:\u002F\u002Fnuxt.com\u002Fdocs"," keeps ",[81,208,205],{},[81,210,211],{},"https:\u002F\u002Fnuxt.com\u002Fdocs\u002Fgetting-started",", but not ",[81,214,215],{},"https:\u002F\u002Fnuxt.com\u002Fdocsearch",". The next character after the prefix has to be ",[81,218,219],{},"\u002F",", ",[81,222,223],{},"?"," or ",[81,226,227],{},"#",".",[74,230,231,232,234,235,237,238,241,242,245],{},"With ",[81,233,201],{}," it's a glob over the whole URL, and ",[81,236,201],{}," matches anything, including nothing. ",[81,239,240],{},"*nuxt.com\u002Fdocs*"," catches both schemes and ",[81,243,244],{},"www."," in one go. Matching ignores case.",[247,248,250,251,254,255,258,259,261],"tip",{"title":249},"Mind the scheme","Wayback writes plenty of old URLs as ",[81,252,253],{},"http:\u002F\u002F…:80\u002F",". A prefix that starts with ",[81,256,257],{},"https:\u002F\u002F"," won't see them. A glob that starts with ",[81,260,201],{}," will.",[86,263,265],{"id":264},"substrings","Substrings",[74,267,268,271,272,275],{},[81,269,270],{},"match"," keeps a URL that contains at least one of the strings. ",[81,273,274],{},"filter"," drops a URL that contains any. Both ignore case. Quick and blunt, and often all you need.",[127,277,279],{"className":129,"code":278,"language":131,"meta":132,"style":132},"await urlscan.discover(\"example.com\", { match: [\"api\", \"admin\"], filter: [\"logout\"] });\n",[81,280,281],{"__ignoreMap":132},[136,282,283,285,288,290,292,295,298,301,303,306,309,312],{"class":138,"line":139},[136,284,143],{"class":142},[136,286,287],{"class":146}," urlscan.",[136,289,151],{"class":150},[136,291,154],{"class":146},[136,293,294],{"class":157},"\"example.com\"",[136,296,297],{"class":146},", { match: [",[136,299,300],{"class":157},"\"api\"",[136,302,220],{"class":146},[136,304,305],{"class":157},"\"admin\"",[136,307,308],{"class":146},"], filter: [",[136,310,311],{"class":157},"\"logout\"",[136,313,314],{"class":146},"] });\n",[86,316,318],{"id":317},"extensions-and-query-keys","Extensions and query keys",[74,320,321,324,325,328,329,228],{},[81,322,323],{},"ext: [\"js\", \"json\", \"bak\"]"," keeps URLs whose path ends in one of those extensions. The extension comes from the last path segment, so ",[81,326,327],{},"\u002Fapp.js?v=3"," counts as ",[81,330,331],{},"js",[74,333,334,337,338,341,342,345],{},[81,335,336],{},"hasQuery: true"," keeps URLs that still have a query after tracking keys are gone. ",[81,339,340],{},"?utm_source=x"," alone doesn't count, ",[81,343,344],{},"?id=7"," does. Handy when you're hunting for parameters.",[86,347,349],{"id":348},"dates","Dates",[74,351,352,96,355,358,359,362,363,366,367,96,370,373],{},[81,353,354],{},"from",[81,356,357],{},"to"," bound the capture date, both ends included. They take archive digits or an ISO date: ",[81,360,361],{},"from: \"2019\""," starts on the first second of 2019, ",[81,364,365],{},"to: \"2019\""," ends on the last one. ",[81,368,369],{},"20190315",[81,371,372],{},"2019-03-15"," work too.",[74,375,376],{},"A bad bound is rejected before any request goes out.",[192,378,380],{"id":379},"what-happens-to-urls-without-a-date","What happens to URLs without a date?",[74,382,383],{},"They stay. AlienVault OTX, URLScan and VirusTotal don't report when they saw a URL, and a date filter doesn't punish them for it. If you want dated URLs only, ask the archives: Wayback, Arquivo.pt, Vefsafn, Common Crawl.",[86,385,387],{"id":386},"order-of-the-checks","Order of the checks",[74,389,390,391,220,393,395,396,96,398,400,401,96,404,228],{},"For each URL: date window, then dedup, then host scope, ",[81,392,120],{},[81,394,124],{},", extension, query keys, and finally ",[81,397,274],{},[81,399,270],{},". The first URL of a kind wins, later duplicates only widen its ",[81,402,403],{},"firstSeen",[81,405,406],{},"lastSeen",[86,408,410],{"id":409},"from-the-cli","From the CLI",[74,412,413],{},"Every option has a flag. Lists are comma separated.",[127,415,419],{"className":416,"code":417,"language":418,"meta":132,"style":132},"language-bash shiki shiki-themes urls urls urls","urls discover nuxt.com -p wayback \\\n  --url-scope 'https:\u002F\u002Fnuxt.com\u002Fdocs' \\\n  --url-out-scope '*\u002F_fonts\u002F*' \\\n  --ext js,json --has-query \\\n  --from 2023 --to 2024\n","bash",[81,420,421,441,451,461,474],{"__ignoreMap":132},[136,422,423,426,429,432,435,438],{"class":138,"line":139},[136,424,425],{"class":150},"urls",[136,427,428],{"class":157}," discover",[136,430,431],{"class":157}," nuxt.com",[136,433,434],{"class":157}," -p",[136,436,437],{"class":157}," wayback",[136,439,440],{"class":146}," \\\n",[136,442,443,446,449],{"class":138,"line":164},[136,444,445],{"class":157},"  --url-scope",[136,447,448],{"class":157}," 'https:\u002F\u002Fnuxt.com\u002Fdocs'",[136,450,440],{"class":146},[136,452,453,456,459],{"class":138,"line":176},[136,454,455],{"class":157},"  --url-out-scope",[136,457,458],{"class":157}," '*\u002F_fonts\u002F*'",[136,460,440],{"class":146},[136,462,463,466,469,472],{"class":138,"line":187},[136,464,465],{"class":157},"  --ext",[136,467,468],{"class":157}," js,json",[136,470,471],{"class":157}," --has-query",[136,473,440],{"class":146},[136,475,477,480,483,486],{"class":138,"line":476},5,[136,478,479],{"class":157},"  --from",[136,481,482],{"class":157}," 2023",[136,484,485],{"class":157}," --to",[136,487,488],{"class":157}," 2024\n",[74,490,491,494],{},[81,492,493],{},"urls discover --help"," lists the short aliases.",[496,497,498],"style",{},"html pre.shiki code .skH_V, html code.shiki .skH_V{--shiki-light:var(--shiki-token-keyword);--shiki-default:var(--shiki-token-keyword);--shiki-dark:var(--shiki-token-keyword)}html pre.shiki code .s38Sx, html code.shiki .s38Sx{--shiki-light:var(--ui-text-highlighted);--shiki-default:var(--ui-text-highlighted);--shiki-dark:var(--ui-text-highlighted)}html pre.shiki code .sK71F, html code.shiki .sK71F{--shiki-light:var(--shiki-token-function);--shiki-default:var(--shiki-token-function);--shiki-dark:var(--shiki-token-function)}html pre.shiki code .shU9J, html code.shiki .shU9J{--shiki-light:var(--shiki-token-string);--shiki-default:var(--shiki-token-string);--shiki-dark:var(--shiki-token-string)}html .light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html.light .shiki span {color: var(--shiki-light);background: var(--shiki-light-bg);font-style: var(--shiki-light-font-style);font-weight: var(--shiki-light-font-weight);text-decoration: var(--shiki-light-text-decoration);}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":132,"searchDepth":164,"depth":164,"links":500},[501,502,505,506,507,510,511],{"id":88,"depth":164,"text":89},{"id":114,"depth":164,"text":115,"children":503},[504],{"id":194,"depth":176,"text":195},{"id":264,"depth":164,"text":265},{"id":317,"depth":164,"text":318},{"id":348,"depth":164,"text":349,"children":508},[509],{"id":379,"depth":176,"text":380},{"id":386,"depth":164,"text":387},{"id":409,"depth":164,"text":410},"Default host scope. URL prefixes and globs. Extensions and query keys and substrings and dates. What each filter keeps and why the limit counts only survivors.","md",null,{"icon":516},"i-lucide-filter",true,{"title":519,"description":512},"Filter discovered URLs by scope and extension and date","4zr77UpKw7i2pWou_MkKMkAkHBj9L1mxsMDc3H0YOyo",[522,524],{"title":12,"path":13,"stem":14,"description":523,"children":-1},"One source by name. The automatic pick with fallback. Or every source side by side with its own results and errors.",{"title":20,"path":21,"stem":22,"description":525,"children":-1},"The urls command. Discover from one source or all of them. Print bare URLs or JSONL. List the sources or start the MCP server.",1790685300951]